HDFS文件读取流程

来源：互联网发布：java静态网页模板编辑：程序博客网时间：2024/05/17 02:48

public class HDFSUtils {/** * TODO: 获取文件系统 * @return */public static FileSystem getFileSystem(){FileSystem hdfs = null ;try{Configuration conf = new Configuration();hdfs = FileSystem.get(conf);}catch(Exception e){e.printStackTrace();}return hdfs;}}

/** * HDFS 通过 FileSystem API 测试类 * */public class HDFSFsTest {//读取文件内容@Testpublic void testRead() throws Exception{//1、获取文件系统FileSystem hdfs = HDFSUtils.getFileSystem();Path path = new Path("/opt/data/test/01.data");//2、打开文件输入流FSDataInputStream inputStream = hdfs.open(path);//读取文件内容到控制台形式  IOUtils.copyBytes(inputStream, System.out, 4096,false); IOUtils.closeStream(inputStream);}//查看目录@Testpublic void testList() throws Exception{FileSystem hdfs = HDFSUtils.getFileSystem();Path path = new Path("/opt/data/");FileStatus[] fileStatus = hdfs.listStatus(path);for(FileStatus fs:fileStatus){Path p = fs.getPath();String info = fs.isDir() ? "目录":"文件";System.out.println(info + " " +p);}} //创建目录@Testpublic void testDirectory() throws Exception{FileSystem hdfs = HDFSUtils.getFileSystem();Path path = new Path("/opt/data/dir");boolean isSuccess = hdfs.mkdirs(path);String info = isSuccess ? "成功":"失败";System.out.println("创建目录【"+ path + "】 " + info);}//上传文件@Testpublic void testPut() throws Exception{FileSystem hdfs = HDFSUtils.getFileSystem();//本地文件 (目录 + 文件名称)Path srcPath = new Path("c:/0125.log");//HDFS文件上传路径Path dstPath = new Path("/opt/data/dir/");//文件上传hdfs.copyFromLocalFile(srcPath, dstPath); }//创建HDFS文件，并写入内容@Testpublic void testCreate() throws Exception{FileSystem hdfs = HDFSUtils.getFileSystem();//HDFS文件上传路径Path path = new Path("/opt/data/dir/touch.data");//创建文件，获取输出流FSDataOutputStream fsDataOutputStream = hdfs.create(path);//写入数据fsDataOutputStream.writeUTF("Hello Hadoop!");IOUtils.closeStream(fsDataOutputStream); }}

core-site.xml

<?xml version="1.0"?><?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <configuration><property><name>fs.default.name</name><value>hdfs://hadoop-master.dragon.org:9000</value></property><property>  <name>hadoop.tmp.dir</name>  <value>/opt/data/tmp</value></property></configuration>

hdfs-site.xml

<?xml version="1.0"?><?xml-stylesheet type="text/xsl" href="configuration.xsl"?><configuration><property><name>dfs.replication</name><value>1</value></property><property><name>dfs.permissions</name><value>false</value></property></configuration>

0 0