Hadoop(05) HDFS Java 接口

来源：互联网发布：linux常用命令编辑：程序博客网时间：2024/06/18 03:53

FileSystem API 简单操作文件

package hadoop.hdfs;import java.io.BufferedInputStream;import java.io.File;import java.io.FileInputStream;import java.io.FileOutputStream;import java.io.InputStream;import java.io.OutputStream;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.junit.Before;import org.junit.Test;public class HDFSTest {    private static FileSystem fileSystem = null;    private final static int buffSize    = 4096;    private final static boolean close   = true;    @Before    public void before() throws Exception {        // 获取通用文件系统        fileSystem = FileSystem.get(new URI("hdfs://Huaqing:9000"),                new Configuration(), "root");    }    @Test    public void _put() throws Exception {        // "/test.c" 代表HDFS / 下的test.c 文件        InputStream in = fileSystem.open(new Path("/test.c"));        OutputStream out = new FileOutputStream(new File("D:/test.c"));        // 原始拷贝文件        // int readBytes = 0;        // byte[] buffer = new byte[4096];        // try {        //     while (-1 != (readBytes = in.read(buffer))) {        //         out.write(buffer);        //     }        // } finally {        //     out.close();        //     in.close();        // }        // hadoop 提供的拷贝工具        IOUtils.copyBytes(in, out, buffSize, close);    }    @Test    public void _get() throws Exception {        boolean overwrite = true;        InputStream in = new BufferedInputStream(                new FileInputStream(new File("D:/pom.xml")));        OutputStream out = fileSystem.create(new Path("/pom.xml"), overwrite);        IOUtils.copyBytes(in, out, buffSize, close);    }    @Test    public void _rm() throws Exception {        boolean recursive = true; // 递归        boolean result = fileSystem.delete(new Path("/del_dir"), recursive);        System.out.println("rm -rf del_dir success " + result);    }    @Test    public void _mkdir() throws Exception {        boolean result = fileSystem.mkdirs(new Path("/del_dir"));        System.out.println("mkdir success " + result);    }    @Test    public void _ls() throws Exception {        FileStatus[] lses = fileSystem.listStatus(new Path("/"));        for (FileStatus fileStatus : lses) {            System.out.println(fileStatus);        }    }}

在windows异常
java.lang.IllegalArgumentException: java.net.UnknownHostException: master
解决：在window下的 C:\Windows\System32\drivers\etc\hosts 配置与hdfs的关系映射。如：192.168.1.103 master

0 0