HDFS之JAVA API

来源：互联网发布：daum potplayer mac版编辑：程序博客网时间：2024/04/30 12:43

一个通过JAVA API操作HDFS文件系统的例子，本例子使用的是hadoop0.20的版本，在windows的eclipse下运行的时候，需要将core-site.xml和hdfs-site.xml放在src/bin目录中。

package com.util;import java.io.BufferedInputStream;import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.apache.hadoop.util.Progressable;public class HdsfAPI {/** 上传文件到HDFS上去 */private static void uploadToHdfs(){String localSrc = "D://mahout.jpg";String dst = "hdfs://wangchen-virtual-machine:9000/input/image2";try {InputStream in = new BufferedInputStream(new FileInputStream(localSrc));Configuration conf = new Configuration();FileSystem fs = FileSystem.get(URI.create(dst), conf);OutputStream out = fs.create(new Path(dst), new Progressable() {public void progress() {System.out.print(".");}});IOUtils.copyBytes(in, out, 4096, true);} catch (FileNotFoundException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();}}/** 从HDFS上读取文件到本地*/private static void readFromHdfs() throws FileNotFoundException,IOException {String dst = "hdfs://localhost:9000/user/wangchen/uploadToHdfs.txt";Configuration conf = new Configuration();FileSystem fs = FileSystem.get(URI.create(dst), conf);FSDataInputStream hdfsInStream = fs.open(new Path(dst));OutputStream out = new FileOutputStream("/home/wangchen/getFromHdfs.txt");byte[] ioBuffer = new byte[1024];int readLen = hdfsInStream.read(ioBuffer);while (-1 != readLen) {out.write(ioBuffer, 0, readLen);readLen = hdfsInStream.read(ioBuffer);}out.close();hdfsInStream.close();fs.close();}/** * 以append方式将内容添加到HDFS上文件的末尾; * 注意：文件更新，需要在hdfs-site.xml中添<property><name>dfs. * append.support</name><value>true</value></property> * 注意：对于append操作，从hadoop-0.21版本开始就不支持了 */private static void appendToHdfs() throws FileNotFoundException,IOException {String dst = "hdfs://localhost:9000/user/wangchen/uploadToHdfs.txt";Configuration conf = new Configuration();FileSystem fs = FileSystem.get(URI.create(dst), conf);FSDataOutputStream out = fs.append(new Path(dst));int readLen = "wangchen append this file !".getBytes().length;while (-1 != readLen) {out.write("wangchen append this file !".getBytes(), 0, readLen);}out.close();fs.close();}/** 从HDFS上删除文件 */private static void deleteFromHdfs() throws FileNotFoundException,IOException {String dst = "hdfs://localhost:9000/user/wangchen/uploadToHdfs.txt";Configuration conf = new Configuration();FileSystem fs = FileSystem.get(URI.create(dst), conf);fs.deleteOnExit(new Path(dst));fs.close();}/** 遍历HDFS上的文件和目录 */private static void getDirectoryFromHdfs() throws FileNotFoundException,IOException {String dst = "hdfs://localhost:9000/user/wangchen";Configuration conf = new Configuration();FileSystem fs = FileSystem.get(URI.create(dst), conf);FileStatus fileList[] = fs.listStatus(new Path(dst));int size = fileList.length;for (int i = 0; i < size; i++) {System.out.println("name:" + fileList[i].getPath().getName()+ "\t\tsize:" + fileList[i].getLen());}fs.close();}public static void main(String[] args) {uploadToHdfs();/*try {uploadToHdfs();//readFromHdfs();//appendToHdfs();//deleteFromHdfs();//getDirectoryFromHdfs();} catch (FileNotFoundException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();}*/}}

操纵的方法有很多种，下面的例子仅供参考，

[java] view plain copy
 
public class TestHDFSFile {  
  
    private String localPath = "C:/D/JavaWorkSpace/bigdata/temp/";  
    private String hdfsPath = "hdfs://192.168.2.6:9000/user/hadoop/temp/";  
  
    public static void main(String[] args) throws Exception {  
        // new TestHDFSFile().testUpload();  
        // new TestHDFSFile().testCreate();  
        //new TestHDFSFile().testRename();  
        //new TestHDFSFile().testDel();  
        //new TestHDFSFile().testgetModifyTime();  
        //new TestHDFSFile().testExists();  
        //new TestHDFSFile().testFileBlockLocation();  
          
        new TestHDFSFile().testGetHostName();  
          
    }  
  
    // 上传本地文件到HDFS  
    public void testUpload() throws Exception {  
  
        Configuration conf = new Configuration();  
        // conf.addResource(new Path(localPath + "core-site.xml"));  
        FileSystem hdfs = FileSystem.get(conf);  
        Path src = new Path(localPath + "file01.txt");  
        Path dst = new Path(hdfsPath);  
        hdfs.copyFromLocalFile(src, dst);  
  
        System.out.println("Upload to " + conf.get("fs.default.name"));  
        FileStatus files[] = hdfs.listStatus(dst);  
        for (FileStatus file : files) {  
            System.out.println(file.getPath());  
        }  
    }  
  
    // 创建HDFS文件  
    public void testCreate() throws Exception {  
        Configuration conf = new Configuration();  
        byte[] buff = "hello world!".getBytes();  
  
        FileSystem hdfs = FileSystem.get(conf);  
        Path dst = new Path(hdfsPath + "hello.txt");  
        FSDataOutputStream outputStream = null;  
        try {  
            outputStream = hdfs.create(dst);  
            outputStream.write(buff, 0, buff.length);  
        } catch (Exception e) {  
            e.printStackTrace();  
  
        } finally {  
            if (outputStream != null) {  
                outputStream.close();  
            }  
        }  
  
        FileStatus files[] = hdfs.listStatus(dst);  
        for (FileStatus file : files) {  
            System.out.println(file.getPath());  
        }  
    }  
  
    // 重命名HDFS文件  
  
    public void testRename() throws Exception {  
  
        Configuration conf = new Configuration();  
  
        FileSystem hdfs = FileSystem.get(conf);  
        Path dst = new Path(hdfsPath);  
  
        Path frpath = new Path(hdfsPath + "hello.txt");  
        Path topath = new Path(hdfsPath + "hello2.txt");  
  
        hdfs.rename(frpath, topath);  
  
        FileStatus files[] = hdfs.listStatus(dst);  
        for (FileStatus file : files) {  
            System.out.println(file.getPath());  
        }  
    }  
  
    // 刪除HDFS文件  
    public void testDel() throws Exception {  
  
        Configuration conf = new Configuration();  
  
        FileSystem hdfs = FileSystem.get(conf);  
        Path dst = new Path(hdfsPath);  
  
        Path topath = new Path(hdfsPath+ "hello2.txt");  
  
        boolean ok = hdfs.delete(topath, false);  
        System.out.println(ok ? "删除成功" : "删除失败");  
  
        FileStatus files[] = hdfs.listStatus(dst);  
        for (FileStatus file : files) {  
            System.out.println(file.getPath());  
        }  
    }  
  
    // 查看HDFS文件的最后修改时间  
    public void testgetModifyTime() throws Exception {  
  
        Configuration conf = new Configuration();  
  
        FileSystem hdfs = FileSystem.get(conf);  
        Path dst = new Path(hdfsPath);  
  
        FileStatus files[] = hdfs.listStatus(dst);  
        for (FileStatus file : files) {  
            System.out.println(file.getPath() + "\t"  
                    + file.getModificationTime());  
              
            System.out.println(file.getPath() + "\t"  
                    + new Date(file.getModificationTime()));  
              
        }  
    }  
  
    // 查看HDFS文件是否存在  
  
    public void testExists() throws Exception {  
  
        Configuration conf = new Configuration();  
          
        FileSystem hdfs = FileSystem.get(conf);  
        Path dst = new Path(hdfsPath + "file01.txt");  
  
        boolean ok = hdfs.exists(dst);  
        System.out.println(ok ? "文件存在" : "文件不存在");  
    }  
  
    // 查看某个文件在HDFS集群的位置  
    public void testFileBlockLocation() throws Exception {  
  
        Configuration conf = new Configuration();  
  
        FileSystem hdfs = FileSystem.get(conf);  
        Path dst = new Path(hdfsPath + "file01.txt");  
  
        FileStatus fileStatus = hdfs.getFileStatus(dst);  
        BlockLocation[] blockLocations = hdfs.getFileBlockLocations(fileStatus,  
                0, fileStatus.getLen());  
        for (BlockLocation block : blockLocations) {  
            System.out.println(Arrays.toString(block.getHosts()) + "\t"  
                    + Arrays.toString(block.getNames()));  
        }  
    }  
  
    // 获取HDFS集群上所有节点名称  
    public void testGetHostName() throws Exception {  
  
        Configuration conf = new Configuration();  
          
        DistributedFileSystem hdfs = (DistributedFileSystem) FileSystem  
                .get(conf);  
        DatanodeInfo[] dataNodeStats = hdfs.getDataNodeStats();  
  
        for (DatanodeInfo dataNode : dataNodeStats) {  
            System.out.println(dataNode.getHostName() + "\t"  
                    + dataNode.getName());  
        }  
    }  
}  

0 0