hadoop 开发API

上文中本人是通过Hadoop的eclipse插件来管理HDFS文件系统的，在实际生产环境通常是用编程的方式来管理hadoop的文件系统

下面我们编程方式开发一个管理hadoop文件系统的应用

首先打开eclipse开发工具，新建Ｍap/Reduce Project项目

点击Next，填写项目名称，并且配置hadoop安装目录

新建立的项目会自动导入hadoop的相关依赖jar文件，不必我们手动添加了

接下来我们调用hadoop文件系统的API来执行文件操作，我们先建立一个文件管理工具类

public class HDFSUtil {        /**     * 从本地复制文件到HDFS     * @param srcFilePath     * @param dstFilePath     * @throws IOException     */    public static void copyFile2HDFS(String srcFilePath,String dstFilePath){        Configuration conf = new Configuration();        Path src = new Path(srcFilePath);        Path dst = new Path(dstFilePath);                        FileSystem hdfs=null;        try {            hdfs = FileSystem.get(conf);                        hdfs.copyFromLocalFile(src, dst);                        FileStatus[] files = hdfs.listStatus(dst);            if(files!=null) {                for(int i=0;i<files.length;i++) {                    System.out.println("the file is:"+files[i].getPath().getName());                }            }else {                System.out.println("no files");            }        } catch (IOException e) {            // TODO Auto-generated catch block            e.printStackTrace();        }        finally        {            try {                hdfs.close();            } catch (IOException e) {                // TODO Auto-generated catch block                e.printStackTrace();            }        }            }        /**     * 在HDFS上创建文件     * @param content     * @param dstFile     * @throws IOException      */    public static void createFileInHDFS(String content,String dstFile) {        Configuration conf = new Configuration();        //conf.addResource(new Path("/home/chenying/hadoop-1.2.0/conf/core-site.xml"));        Path dst = new Path(dstFile);        FileSystem hdfs = null;        FSDataOutputStream out = null;        try {                        hdfs=FileSystem.get(conf);            out = hdfs.create(dst);           // out.writeBytes(content);            out.write(content.getBytes("UTF-8"));            out.flush();        } catch (IOException e) {            e.printStackTrace();                    }finally {            if(hdfs != null) {                try {                    hdfs.close();                } catch (IOException e) {                    // TODO Auto-generated catch block                    e.printStackTrace();                }            }            if(out != null) {                try {                    out.close();                } catch (IOException e) {                    e.printStackTrace();                }            }                    }            }    /**     * 在HDFS上创建目录     * @param content     * @param dstFile     * @throws IOException      */    public static void createDirectoryInHDFS(String dstDir) {        Configuration conf = new Configuration();        Path dst = new Path(dstDir);        FileSystem hdfs = null;                try {                        hdfs=FileSystem.get(conf);            hdfs.mkdirs(dst);                    } catch (IOException e) {            e.printStackTrace();            //throw new IOException(e);        }finally {            if(hdfs != null) {                try {                    hdfs.close();                } catch (IOException e) {                    // TODO Auto-generated catch block                    e.printStackTrace();                }            }        }            }        /**     * 重命名文件     * @param originalFile     * @param newFile     * @throws IOException     */    public static void renameFileInHDFS(String originalFile,String newFile)  {        Configuration conf = new Configuration();        Path originalPath = new Path(originalFile);        Path newPath = new Path(newFile);                FileSystem hdfs=null;        try {            //hdfs = newPath.getFileSystem(conf);            hdfs =FileSystem.get(conf);            boolean isRename=hdfs.rename(originalPath, newPath);            String result=isRename?"成功":"失败";            System.out.println("文件重命名结果为："+result);        } catch (IOException e) {            // TODO Auto-generated catch block            e.printStackTrace();        }        finally        {            try {                hdfs.close();            } catch (IOException e) {                // TODO Auto-generated catch block                e.printStackTrace();            }        }            }        /**     * 获得文件的最后修改时间     * @param dstFile     * @throws IOException     */    public static void getFileLastModifyTime(String dstFile) {        Configuration conf = new Configuration();        Path dstPath = new Path(dstFile);        FileSystem hdfs=null;        try {            //hdfs = dstPath.getFileSystem(conf);            hdfs =FileSystem.get(conf);            FileStatus file = hdfs.getFileStatus(dstPath);            long time = file.getModificationTime();                        System.out.println("the last modify time is : "+new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(time)));        } catch (IOException e) {            // TODO Auto-generated catch block            e.printStackTrace();        }        finally        {            try {                hdfs.close();            } catch (IOException e) {                // TODO Auto-generated catch block                e.printStackTrace();            }        }            }        /**     * 检查文件是否存在     * @param dstFile     * @throws IOException     */    public static void checkFileIsExists(String dstFile) {        Configuration conf = new Configuration();        Path dstPath = new Path(dstFile);        FileSystem hdfs=null;        try {            //hdfs = dstPath.getFileSystem(conf);            hdfs =FileSystem.get(conf);            boolean flag = hdfs.exists(dstPath);                        System.out.println("is the file exists:"+flag);        } catch (IOException e) {            // TODO Auto-generated catch block            e.printStackTrace();        }        finally        {            try {                hdfs.close();            } catch (IOException e) {                // TODO Auto-generated catch block                e.printStackTrace();            }        }            }        /**     * 获得文件的存放位置     * @param dstFile     * @throws IOException     */    public static void getFileLocations(String dstFile){        Configuration conf = new Configuration();        Path dstPath = new Path(dstFile);        FileSystem hdfs=null;        try {            //hdfs = dstPath.getFileSystem(conf);            hdfs =FileSystem.get(conf);            FileStatus file = hdfs.getFileStatus(dstPath);            BlockLocation[] blkLocations = hdfs.getFileBlockLocations(file, 0, file.getLen());            if(blkLocations != null) {                int len = blkLocations.length;                for(int i=0;i<len;i++) {                    String[] hosts = blkLocations[i].getHosts();                    for(String host:hosts) {                        System.out.println("the location'host is : "+host);                    }                }            }        } catch (IOException e) {            // TODO Auto-generated catch block            e.printStackTrace();        }        finally        {            try {                hdfs.close();            } catch (IOException e) {                // TODO Auto-generated catch block                e.printStackTrace();            }        }    }        /**     * 删除文件     * @throws IOException     */    public static void deleteFile(String dstFile){        Configuration conf = new Configuration();        Path dstPath = new Path(dstFile);        FileSystem hdfs=null;        try {            //hdfs = dstPath.getFileSystem(conf);            hdfs =FileSystem.get(conf);            boolean flag = hdfs.delete(dstPath, true);////如果是目录则递归删除            System.out.println("is deleted : "+flag);        } catch (IOException e) {            // TODO Auto-generated catch block            e.printStackTrace();        }        finally        {            try {                hdfs.close();            } catch (IOException e) {                // TODO Auto-generated catch block                e.printStackTrace();            }        }           }}

（如果你愿意折腾，可以尝试将上面的方法改造成接口回调，本人就不做这个无谓的折腾了）

如果遇到Exception in thread "main" java.lang.IllegalArgumentException: Wrong FS: hdfs expected: file:///之类的异常，需要将${hadoop.root}/conf/目录的hadoop配置文件core-site.xml、hdfs-site.xml、mapred-site.xml拷贝到eclipse项目的bin目录里面

下面是测试类

public class HdfsClient {        /**     * 上传文件     * @throws IOException     */    @Test    public void testCopyFile2HDFS() throws IOException {        String srcFilePath = "/home/chenying/FetchTask.java";        String dstFilePath = "hdfs://localhost:9000/user/coder/in";        HDFSUtil.copyFile2HDFS(srcFilePath, dstFilePath);    }    /**     * 创建文件     */    @Test    public void testCreateFileInHDFS()  {        String content = "hey,Hadoop. 测试中文";        String dstFile = "hdfs://localhost:9000/user/coder/input/hey";        HDFSUtil.createFileInHDFS(content, dstFile);    }    /**     * 重命名文件     */    @Test    public void testRenameFileInHDFS() {        String originalFile = "hdfs://localhost:9000/user/coder/input/hey";        String newFile = "hdfs://localhost:9000/user/coder/input/hey_hadoop";        HDFSUtil.renameFileInHDFS(originalFile, newFile);    }    /**     * 获取文件的最后修改时间     */    @Test    public void testGetFileLastModifyTimme() {        String dstFile = "hdfs://localhost:9000/user/coder/input/hey_hadoop";        HDFSUtil.getFileLastModifyTime(dstFile);    }    /**     * 检查文件是否存在     */    @Test    public void testCheckFileIsExists() {        String dstFile = "hdfs://localhost:9000/user/coder/input/hey_hadoop";        HDFSUtil.checkFileIsExists(dstFile);    }    /**     * 获取文件的存放位置     */    @Test    public void testGetFileLocations() {        String dstFile = "hdfs://localhost:9000/user/coder/input/hey_hadoop";        HDFSUtil.getFileLocations(dstFile);    }    /**     * 删除文件     */    @Test    public void testDeleteFile(){        String dstFile = "hdfs://localhost:9000/user/coder/output";        HDFSUtil.deleteFile(dstFile);    }}

运行该测试类，正常情况下，上面的测试用例会显示绿色

后面部分，本人打算采用HDFS开发一个图片服务器 

--------------------------------------------------------------------------- 

本系列Hadoop1.2.0开发笔记系本人原创 

转载请注明出处 博客园 刺猬的温驯  

本文链接 http://www.cnblogs.com/chenying99/archive/2013/06/02/3113449.html