Hadoop基础教程-第4章 HDFS的Java API（4.7 Java API封装类）（草稿）

第4章 HDFS的Java API

4.7 Java API封装类

package cn.hadron.hdfsDemo;import java.io.IOException;import java.io.InputStream;import java.net.URI;import java.net.URISyntaxException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.apache.commons.lang.StringUtils;/** * HDFS工具类 * @author 程裕强 *  */public class HDFSUtil {    /*     * 获取文件系统     */    public static FileSystem getFileSystem(String url) {        //StringUtils中方法的操作对象是java.lang.String类型的对象，是JDK提供的String类型操作方法的补充        if(StringUtils.isBlank(url)){//判断某字符串是否为空或长度为0或由空白符(whitespace)构成             return null;        }        Configuration conf = new Configuration();        FileSystem fs = null;        try {            URI uri = new URI(url.trim());            fs = FileSystem.get(uri,conf);        } catch (URISyntaxException|IOException e) {            System.out.println(e);        }        return fs;    }    /*     * 获取文件系统     */    public static FileSystem getFileSystem(String url,String user) {        if(StringUtils.isBlank(url)){            return null;        }        Configuration conf = new Configuration();        FileSystem fs = null;        try {            URI uri = new URI(url.trim());            fs = FileSystem.get(uri,conf,user);        } catch (InterruptedException|URISyntaxException|IOException e) {            System.out.println(e);        }        return fs;    }    /*     * 创建目录     */    public static boolean mkdir(String path) throws Exception{        FileSystem fs = getFileSystem(path,"root");        boolean b=fs.mkdirs(new Path(path));        fs.close();        return b;    }    /**     * 读文件     * @param filePath     * @throws IOException     */    public static void readFile(String filePath) throws IOException{        FileSystem fs = getFileSystem(filePath);        InputStream in=null;        try{            in=fs.open(new Path(filePath));            IOUtils.copyBytes(in, System.out,4096,false);        }catch(Exception e){            System.out.println(e.getMessage());        }finally{            IOUtils.closeStream(in);        }    }    /**     * 上传文件     * @param src     * @param dst     * @throws IOException     */    public static void putFile(String localPath,String hdfsPath) throws IOException{        FileSystem fs = getFileSystem(hdfsPath,"root");        fs.copyFromLocalFile(new Path(localPath), new Path(hdfsPath));        fs.close();    }     /**     * 下载文件     * @param hdfsPath     * @param localPath     * @throws IOException     */    public static void getFile(String hdfsPath,String localPath) throws IOException{        FileSystem fs = getFileSystem(hdfsPath,"root");        Path hdfs_path = new Path(hdfsPath);        Path local_path = new Path(localPath);        fs.copyToLocalFile(hdfs_path, local_path);        fs.close();    }    /**     * 递归删除     * @param hdfsPath     * @return     * @throws IllegalArgumentException     * @throws IOException     */    public static boolean deleteFile(String hdfsPath) throws IllegalArgumentException, IOException{        FileSystem fs = getFileSystem(hdfsPath,"root");        return fs.delete(new Path(hdfsPath), true);    }    /**     * 目录列表     * @param hdfsPath     * @return     */    public static String[] listFile(String hdfsPath){        String[] files = new String[0];        FileSystem fs = getFileSystem(hdfsPath,"root");        Path path=new Path(hdfsPath);        FileStatus[] st=null;        try {            st = fs.listStatus(path);        } catch (FileNotFoundException e) {            e.printStackTrace();        } catch (IOException e) {            e.printStackTrace();        }        files = new String[st.length];        for(int i=0;i<st.length;i++){            files[i]=st[i].toString();        }        return files;    }    /**     * 主方法，测试     * @param args     * @throws Exception     */    public static void main(String[] args) throws Exception{        String base="hdfs://192.168.80.131:9000/";        HDFSUtil.mkdir(base+"util");        HDFSUtil.putFile("D:\\words", base+"util/");        HDFSUtil.readFile(base+"util/words/words.txt");        HDFSUtil.getFile(base+"util/words", "D:\\util");        HDFSUtil.deleteFile(base+"abc");    }}

执行main方法前

[root@node1 ~]# hdfs dfs -ls /Found 2 itemsdrwxr-xr-x   - root supergroup          0 2017-05-14 09:40 /abcdrwxr-xr-x   - root supergroup          0 2017-05-14 09:37 /user

执行main方法后

[root@node1 ~]# hdfs dfs -ls /Found 2 itemsdrwxr-xr-x   - root supergroup          0 2017-05-14 09:37 /userdrwxr-xr-x   - root supergroup          0 2017-05-21 09:07 /util[root@node1 ~]# hdfs dfs -ls /util/wordsFound 2 items-rw-r--r--   3 root supergroup         30 2017-05-21 09:18 /util/words/word2.txt-rw-r--r--   3 root supergroup         55 2017-05-21 09:18 /util/words/words.txt

修改主方法

public static void main(String[] args) throws Exception{        String base="hdfs://192.168.80.131:9000/";    /*  HDFSUtil.mkdir(base+"util");        HDFSUtil.putFile("D:\\words", base+"util/");        HDFSUtil.readFile(base+"util/words/words.txt");        HDFSUtil.getFile(base+"util/words", "D:\\util");        HDFSUtil.deleteFile(base+"abc");*/        String[] array=HDFSUtil.listFile(base);        for(String s:array){            System.out.println(s);        }    }

Eclipse输出

log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory).log4j:WARN Please initialize the log4j system properly.log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.FileStatus{path=hdfs://192.168.80.131:9000/user; isDirectory=true; modification_time=1494769072046; access_time=0; owner=root; group=supergroup; permission=rwxr-xr-x; isSymlink=false}FileStatus{path=hdfs://192.168.80.131:9000/util; isDirectory=true; modification_time=1495372718309; access_time=0; owner=root; group=supergroup; permission=rwxr-xr-x; isSymlink=false}