使用JavaAPI操作hadoop hdfs <一>

package com.zn.hadoop;import java.net.URI;import java.util.Iterator;import java.util.Map.Entry;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.LocatedFileStatus;import org.apache.hadoop.fs.Path;import org.apache.hadoop.fs.RemoteIterator;import org.junit.Before;import org.junit.Test;/** * 1、客户端去操作hdfs时，是有一个用户身份的 * 默认情况下，hdfs客户端api会从jvm中获取一个参数来作为自己的用户身份：-DHADOOP_USER_NAME=hadoop * 也可以在构造客户端fs对象时，通过参数传递进去  * 2、当org.apache.hadoop.security.AccessControlException: Permission denied 报该异常时 * 我们需要修改 hadoop hdfs根目录读写权限  hadoop fs -chmod 777 / *  * @author ning */public class HdfsClientDemo {FileSystem fs = null;Configuration conf = null;@Beforepublic void init() throws Exception {conf = new Configuration();conf.set("fs.defaultFS", "hdfs://Master.Hadoop:9000", "hadoop");// 拿到一个文件系统操作的客户端实例对象/* fs = FileSystem.get(conf); */// 可以直接传入 uri和用户身份fs = FileSystem.get(new URI("hdfs://Master.Hadoop:9000"), conf); // 最后一个参数为用户名}@Testpublic void testUpload() throws Exception {Thread.sleep(2000);fs.copyFromLocalFile(new Path("/Users/ning/Downloads/hadoop.test"), new Path("/access.log.copy"));fs.close();}@Testpublic void testDownload() throws Exception {fs.copyToLocalFile(new Path("/access.log.copy"), new Path("/Users/ning/Downloads/hadoop2.test"));fs.close();}@Testpublic void testConf() {Iterator<Entry<String, String>> iterator = conf.iterator();while (iterator.hasNext()) {Entry<String, String> entry = iterator.next();System.out.println(entry.getValue() + "--" + entry.getValue());// conf加载的内容}}/** * 创建目录 */@Testpublic void makdirTest() throws Exception {boolean mkdirs = fs.mkdirs(new Path("/aaa/bbb"));System.out.println(mkdirs);}/** * 删除 */@Testpublic void deleteTest() throws Exception {boolean delete = fs.delete(new Path("/aaa"), true);// true， 递归删除System.out.println(delete);}@Testpublic void listTest() throws Exception {FileStatus[] listStatus = fs.listStatus(new Path("/"));for (FileStatus fileStatus : listStatus) {System.err.println(fileStatus.getPath() + "=================" + fileStatus.toString());}// 会递归找到所有的文件RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);while (listFiles.hasNext()) {LocatedFileStatus next = listFiles.next();String name = next.getPath().getName();Path path = next.getPath();System.out.println(name + "---" + path.toString());}}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();conf.set("fs.defaultFS", "hdfs://Master.Hadoop:9000");// 拿到一个文件系统操作的客户端实例对象FileSystem fs = FileSystem.get(conf);fs.copyFromLocalFile(new Path("/Users/ning/Downloads/hadoop.test"), new Path("/access.log.copy"));fs.close();}}