调用java api 访问HDFS

来源：互联网发布：剑桥战争史知乎编辑：程序博客网时间：2024/06/05 17:33

依赖

<dependency><groupId> org.apache.hadoop</groupId><artifactId>hadoop-client </artifactId><version>2.6.0</version></dependency>

上传文件

/**     * 上传文件到hdfs系统     * 伪分布方式启动下的实现     * @param s 源文件     * @param d 目标路径     * @throws IOException     * @see [类、类#方法、类#成员]     */    public static void uploadLocalFile2HDFS(String s, String d)        throws IOException    {                Configuration config = new Configuration();        Path dstDir = new Path(d);         FileSystem hdfs = dstDir.getFileSystem(config);                Path src = new Path(s);                  hdfs.copyFromLocalFile(src, dstDir);                hdfs.close();    }

创建文件并写入内容

/**     * 创建文件并写入内容     * <功能详细描述>     * @param toCreateFilePath 文件名【可以指定路径】     * @param content 文件内容     * @param hdfsAddress 连接的HDFS地址     * @throws IOException     * @see [类、类#方法、类#成员]     */    public static void createNewHDFSFile(String toCreateFilePath, String content,String hdfsAddress) throws IOException    {        Configuration config = new Configuration();        Path dstDir = new Path(hdfsAddress);         FileSystem hdfs = dstDir.getFileSystem(config);                FSDataOutputStream os = hdfs.create(new Path(toCreateFilePath));        os.write(content.getBytes("UTF-8"));                os.close();                hdfs.close();    }

删除文件/目录

/**     * 删除文件/目录     * <功能详细描述>     * @param dst     * @param hdfsAddress     * @return     * @throws IOException     * @see [类、类#方法、类#成员]     */    public static boolean deleteHDFSFile(String dst,String hdfsAddress) throws IOException    {        Configuration config = new Configuration();        Path dstDir = new Path(hdfsAddress);         FileSystem hdfs = dstDir.getFileSystem(config);                Path dstPath = new Path(dst);         boolean isDeleted = hdfs.deleteOnExit(dstPath);                hdfs.close();                return isDeleted;    }

读取文件

/**     * 读取文件     * <功能详细描述>     * @param dst     * @param hdfsAddress     * @return     * @throws Exception     * @see [类、类#方法、类#成员]     */    public static byte[] readHDFSFile(String dst,String hdfsAddress) throws Exception    {        Configuration config = new Configuration();        Path dstDir = new Path(hdfsAddress);         FileSystem fs = dstDir.getFileSystem(config);                // check if the file exists        Path path = new Path(dst);        if ( fs.exists(path) )        {            FSDataInputStream is = fs.open(path);            // get the file info to create the buffer            FileStatus stat = fs.getFileStatus(path);                        // create the buffer            byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat.getLen()))];            is.readFully(0, buffer);                        is.close();            fs.close();                        return buffer;        }        else        {            throw new Exception("the file is not found .");        }    }

创建目录

/**     * 创建目录及文件     * 可递归的创建多层目录     * @param dir     * @param hdfsAddress     * @throws IOException     * @see [类、类#方法、类#成员]     */    public static void mkdir(String dir,String hdfsAddress) throws IOException    {        Configuration config = new Configuration();        Path dstDir = new Path(hdfsAddress);         FileSystem fs = dstDir.getFileSystem(config);                fs.mkdirs(new Path(dir));                fs.close();    }

读取目录下的所有文件

/**     * 展示目录下的文件和文件夹     * <功能详细描述>     * @param dir     * @param hdfsAddress     * @throws IOException     * @see [类、类#方法、类#成员]     */    public static void listAll(String dir,String hdfsAddress) throws IOException    {        Configuration config = new Configuration();        Path dstDir = new Path(hdfsAddress);         FileSystem fs = dstDir.getFileSystem(config);                FileStatus[] stats = fs.listStatus(new Path(dir));                for(int i = 0; i < stats.length; ++i)        {            if (stats[i].isFile())            {                // regular file                System.out.println(stats[i].getPath().toString());            }            else if (stats[i].isDirectory())            {                // dir                System.out.println(stats[i].getPath().toString());            }            else if(stats[i].isSymlink())            {                // is s symlink in linux                System.out.println(stats[i].getPath().toString());            }                         }        fs.close();    }

将本地文件内容copy到hdfs文件中

/**     * 将本地文件内容copy到hdfs文件中     * <功能详细描述>     * @param localFilePath 本地文件路径     * @param hdfsPath hdfs文件路径     * @throws FileNotFoundException      * @throws IOException     * @see [类、类#方法、类#成员]     */    public static void FileCopyWhitProcess(String localFilePath, String hdfsPath) throws FileNotFoundException{        InputStream in = new BufferedInputStream(new FileInputStream(localFilePath));        Configuration config = new Configuration();        FileSystem fs;        FSDataOutputStream out;        try        {                       fs = FileSystem.get(URI.create(hdfsPath), config);            out= fs.create(new Path(hdfsPath), new Progressable()            {                                @Override                public void progress()                {                    System.out.println(".");                                    }            });            IOUtils.copyBytes(in, out, 4096, true);        }        catch (IOException e)        {            // TODO Auto-generated catch block            e.printStackTrace();        }finally{            IOUtils.closeStream(in);        }            }

0 0