用流的方式来操作hdfs上的文件

来源：互联网发布：淘宝网账户登录编辑：程序博客网时间：2024/05/13 01:54
package cn.itcast.bigdata.hdfs;import java.io.FileInputStream;import java.io.FileOutputStream;import java.io.IOException;import java.net.URI;import org.apache.commons.io.IOUtils;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.junit.Before;import org.junit.Test;/** * 用流的方式来操作hdfs上的文件 * 可以实现读取指定偏移量范围的数据 * @author * */public class HdfsStreamAccess {FileSystem fs = null;Configuration conf = null;@Beforepublic void init() throws Exception{conf = new Configuration();//拿到一个文件系统操作的客户端实例对象//fs = FileSystem.get(conf);//可以直接传入 uri和用户身份fs = FileSystem.get(new URI("hdfs://mini1:9000"),conf,"hadoop");}/** * 通过流的方式上传文件到hdfs * @throws Exception */@Testpublic void testUpload() throws Exception {FSDataOutputStream outputStream = fs.create(new Path("/angelababy.love"), true);FileInputStream inputStream = new FileInputStream("c:/angelababy.love");IOUtils.copy(inputStream, outputStream);}/** * 通过流的方式获取hdfs上数据 * @throws Exception */@Testpublic void testDownLoad() throws Exception {FSDataInputStream inputStream = fs.open(new Path("/angelababy.love"));FileOutputStream outputStream = new FileOutputStream("d:/angelababy.love");IOUtils.copy(inputStream, outputStream);}@Testpublic void testRandomAccess() throws Exception{FSDataInputStream inputStream = fs.open(new Path("/angelababy.love"));inputStream.seek(12);FileOutputStream outputStream = new FileOutputStream("d:/angelababy.love.part2");IOUtils.copy(inputStream, outputStream);}/** * 显示hdfs上文件的内容 * @throws IOException  * @throws IllegalArgumentException  */@Testpublic void testCat() throws IllegalArgumentException, IOException{FSDataInputStream in = fs.open(new Path("/angelababy.love"));IOUtils.copy(in, System.out);//IOUtils.copyBytes(in, System.out, 1024);}}
0 0