Hadoop的基本使用（2）——通过代码操作HDFS

来源：互联网发布：mac怎样建立文件夹编辑：程序博客网时间：2024/06/01 10:17

1、导入相关jar包

hadoop-common 相关jar包
hadoop-hdfs 相关jar包

2、指定hadoop入口获得fileSystem对象

public void before() throws IOException{    Configuration conf=new Configuration();    conf.set("fs.defaultFS", "hdfs://192.168.11.128:9999");    conf.setInt("dfs.replication", 1);    fileSystem=FileSystem.get(conf);}

3、通过fileSystem对象对hdfs进行操作

//创建文件夹@Testpublic void mkdir() throws IOException{    Path path=new Path("/demo/test2");    boolean mkdirs = fileSystem.mkdirs(path);    System.out.println(mkdirs);}//判断路径是否存在@Testpublic void exist() throws IOException{    Path path=new Path("/gjhg");    boolean exists = fileSystem.exists(path);    System.out.println(exists);}//判断所给路径是否为文件夹或文件@Testpublic void isFile() throws IOException{    Path src=new Path("/demo/test/helloworld.txt");    boolean dir = fileSystem.isDirectory(src);    boolean file = fileSystem.isFile(src);    System.out.println(dir);    System.out.println(file);}//上传文件：1、路径 2、流@Testpublic void copyFromLocalFile() throws IOException{    Path src=new Path("H:\\hadoopTest\\text.txt");    Path dst=new Path("/demo/test2/text");    fileSystem.copyFromLocalFile(src, dst);}@Testpublic void copyFormLocalStream() throws IOException{    Path dst=new Path("/demo/test/hahah.txt");    FSDataOutputStream fos=fileSystem.create(dst);    FileInputStream fis=new FileInputStream("H:\\hadoopTest\\howareyou.txt");    IOUtils.copyBytes(fis, fos, 1024);    IOUtils.closeStream(fis);    IOUtils.closeStream(fos);}//下载文件：1、路径 2、流@Testpublic void copyToLocalFile() throws IOException{    Path src=new Path("/demo/test/helloworld.txt");    Path dst=new Path("H:/hadoopTest/hi.txt");    fileSystem.copyToLocalFile(false, src, dst, true);}@Testpublic void copyToLocalStream() throws IOException{    Path dst=new Path("/demo/test/hahah.txt");    InputStream fis=fileSystem.open(dst);    FileOutputStream fos=new FileOutputStream("H:\\hadoopTest\\hah.txt");    IOUtils.copyBytes(fis, fos, 1024);    IOUtils.closeStream(fis);    IOUtils.closeStream(fos);}//输出文件信息@Testpublic void fileList() throws FileNotFoundException, IOException{    Path dst=new Path("/");    RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(dst, true);    while(listFiles.hasNext()){    LocatedFileStatus next=listFiles.next();    System.out.println(next);}

4、关闭fileSystem

@Afterpublic void after() throws IOException{    fileSystem.close();}

0 0