HDFS的java操作

HDFS的客户端操作分为 Java和shell命令行

hdfs在生产应用中主要是客户端的开发，其核心步骤是从hdfs提供的api中构造一个HDFS的访问客户端对象，然后通过该客户端对象操作（增删改查）HDFS上的文件

1. 文件的增删改查

 package cn.yzx.bigdata.hdfs;import java.io.FileNotFoundException;import java.io.IOException;import java.util.Iterator;import java.util.Map.Entry;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.BlockLocation;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.LocatedFileStatus;import org.apache.hadoop.fs.Path;import org.apache.hadoop.fs.RemoteIterator;import org.apache.hadoop.hdfs.DFSClient.Conf;import org.junit.Before;import org.junit.Test;public class HdfsClientDemo {FileSystem fs=null;Configuration conf = null;@Beforepublic void init() throws IOException { conf = new Configuration();  conf.set("fs.defaultFS", "hdfs://hadoop01:9000");  conf.set("dfs.defaultFS", "5");  //拿到一个文件系统操作的客户端实例对象 fs = FileSystem.get(conf);}/* * 上传文件 */@Testpublic void testUpload() throws Exception {fs.copyFromLocalFile(new Path("c:/AMTAG.BIN"), new Path("/AMTAG.BIN.copy"));fs.close();}/* * 打印参数 */@Testpublic void testconf() {Iterator<Entry<String, String>> iterator = conf.iterator();while(iterator.hasNext()) {Entry<String, String> next = iterator.next();System.out.println(next.getKey()+" :  "+next.getValue());}}/* * 创建文件夹 */@Testpublic void testMkdir() throws Exception {boolean mkdirs = fs.mkdirs(new Path("/testmkdirs/aaa"));System.out.println(mkdirs);}/* * 删除 */@Testpublic void testdelete() throws Exception {boolean delete = fs.delete(new Path("/testmkdirs/aaa"), true);System.out.println(delete);}/* * 查看 递归 */@Testpublic void testLs() throws Exception {RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);while(files.hasNext()) {LocatedFileStatus fileStatus = files.next();System.out.println("BlockSize   "+fileStatus.getBlockSize());System.out.println("Owner   "+fileStatus.getOwner());System.out.println("Replication   "+fileStatus.getReplication());System.out.println("Permission   "+fileStatus.getPermission());System.out.println("name   "+fileStatus.getPath().getName());BlockLocation[] blockLocations = fileStatus.getBlockLocations();for(BlockLocation blockLocation:blockLocations) {System.out.println("块的ID  "+blockLocation.getNames());System.out.println("块起始偏移量  "+blockLocation.getOffset());System.out.println("datanode "+blockLocation.getLength());String[] datanodes = blockLocation.getHosts();for(String s:datanodes) {System.out.println("datanode "+s);}}System.out.println("+++++++++++++++++++++++++++");}}/* * 查看不用递归 */@Testpublic void testLs2() throws Exception {FileStatus[] listStatus = fs.listStatus(new Path("/"));for(FileStatus s:listStatus) {System.out.println("name "+s.getPath().getName());System.out.println(s.isFile()?"f":"dir");}}}

2 。通过流的方式访问hdfs

hdfs支持随机定位进行文件读取，而且可以方便地读取指定长度

package cn.yzx.bigdata.hdfs;import java.io.FileInputStream;import java.io.FileOutputStream;import java.io.IOException;import org.apache.commons.io.IOUtils;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.junit.Before;import org.junit.Test;/* * 用流的方式操作HDFS上的文件 * 可以指定偏移量的范围的数据 */public class HdfsStreamAccess {FileSystem fs=null;Configuration conf = null;@Beforepublic void init() throws IOException { conf = new Configuration();  conf.set("fs.defaultFS", "hdfs://hadoop01:9000");  //拿到一个文件系统操作的客户端实例对象 fs = FileSystem.get(conf);}@Testpublic void testUpload() throws Exception { //上传文件 anglebaby.love  true表示如果有就覆盖FSDataOutputStream outputstream = fs.create(new Path("/anglababy.txt"),true);FileInputStream inputStream = new FileInputStream("c:/anglababy.txt");IOUtils.copy(inputStream, outputstream);}/* * 从指定位置开始读 */@Testpublic  void testRandAccess () throws IllegalArgumentException, IOException {FSDataInputStream inputStream = fs.open(new Path("/anglababy.txt"));inputStream.seek(12);FileOutputStream outputStream = new FileOutputStream("f:/anglababy.txt");IOUtils.copy(inputStream, outputStream); }}