HDFS For Java

来源：互联网发布：hdfs java创建目录编辑：程序博客网时间：2024/06/11 00:15
package cn.hwadee.wuyang.hdfs.utils;import java.io.ByteArrayOutputStream;import java.io.IOException;import java.io.OutputStream;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;public class ConfigurationUtil {    private static String url = "hdfs://192.168.193.201:9000";    private static String user = "wuyang";    private static FileSystem fs;    /**     * 获取文件系统     * @return     */    public static FileSystem getFileSystem(){        Configuration conf = new Configuration();        try {            return FileSystem.get(URI.create(url), conf, user);        } catch (Exception e) {            return null;        }    }    /**     * 创建文件夹     * @param path     * @throws IOException      */    public static Boolean createFileDir(Path path) throws IOException{        Configuration conf = new Configuration();        try {            fs = FileSystem.get(URI.create(url), conf, user);            if(!fs.exists(path)){                fs.mkdirs(path);                System.out.println(path.getName()+"文件夹创建成功！！！");                return true;            }            return false;        } catch (Exception e) {            e.getMessage();            return false;        }finally {            fs.close();        }    }    /**     * 删除文件     * @param path     * @throws IOException      */    public static Boolean deleteFile(Path path) throws IOException{        Configuration conf = new Configuration();        try {            fs = FileSystem.get(URI.create(url), conf, user);            if(fs.exists(path)){                fs.delete(path, true);                System.out.println(path.getName()+"文件删除成功！！！");                return true;            }            return false;        } catch (Exception e) {            e.getMessage();            return false;        }finally {            fs.close();        }    }    /**     * 重命名     * @param name1     * @param name2     * @throws IOException      */    public static Boolean fileRename(Path name1,Path name2) throws IOException {        Configuration conf = new Configuration();        try {            fs = FileSystem.get(URI.create(url), conf, user);            if (fs.exists(name1)) {                fs.rename(name1, name2);                System.out.println(name2.getName()+"替换"+name1.getName()+"成功!!!");                return true;            }            return false;        } catch (Exception e) {            e.getMessage();            return false;        }finally {            fs.close();        }    }    /**     * 创建文件     * @param path     * @return     */    public static Boolean createFile(Path path){        Configuration conf = new Configuration();        try {            fs = FileSystem.get(URI.create(url), conf, user);            if(!fs.exists(path)){                fs.create(path);                System.out.println(path.getName()+"文件创建成功！！！");                return true;            }else{                System.out.println(path.getName()+"文件已经存在！！！");                return false;            }        }catch (Exception e) {            e.getMessage();            return false;        }finally {            try {                fs.close();            } catch (IOException e) {                e.getMessage();            }        }    }    /**     * 获取path下文件     * @param path     * @return     */    public static FileStatus[] listFile(Path path){        Configuration conf = new Configuration();        try {            fs = FileSystem.get(URI.create(url), conf, user);            if(fs.exists(path)){                FileStatus[] list = fs.listStatus(path);                System.out.println("文件长度:"+list.length);                return list;            }            return null;        } catch (Exception e) {            e.getMessage();            return null;        }       }    /**     * 从hdfs上下载文件     * @param hdfspath     * @param locationpath     * @return     */    public static Boolean downLoadFileFromHDFS(Path hdfspath,Path locationpath){        Configuration conf = new Configuration();        try {            fs = FileSystem.get(URI.create(url), conf, user);            if(fs.exists(hdfspath)){                fs.copyToLocalFile(hdfspath, locationpath);                System.out.println("download: from" + hdfspath.getName() + " to " + locationpath.getName());                return true;            }            return false;        } catch (Exception e) {            e.getMessage();            return false;        }finally {            try {                fs.close();            } catch (IOException e) {                e.getMessage();            }        }       }     /**     * 拷贝文件到HDFS     * @param local     * @param remote     * @throws IOException     */    public static Boolean copyFileToHDFS(String local, String remote) throws IOException {        Configuration conf = new Configuration();        try {            fs = FileSystem.get(URI.create(url), conf, user);            fs.copyFromLocalFile(new Path(local), new Path(remote));            System.out.println("copy from: " + local + " to " + remote);            return true;        } catch (Exception e) {            e.getMessage();            return false;        }finally {            try {                fs.close();            } catch (IOException e) {                e.getMessage();            }        }       }    /**     * 查看文件中的内容     * @param remoteFile     * @return     * @throws IOException     * @throws InterruptedException      */    public static String catFile(String remoteFile) throws IOException, InterruptedException {        Path path = new Path(remoteFile);        Configuration conf = new Configuration();        fs = FileSystem.get(URI.create(url), conf, user);        FSDataInputStream fsdis = null;        OutputStream baos = new ByteArrayOutputStream();        String str = null;        try {            fsdis = fs.open(path);            IOUtils.copyBytes(fsdis, baos, 4096, false);            str = baos.toString();        } finally {            IOUtils.closeStream(fsdis);            fs.close();        }        return str;    }}
阅读全文
2 0