java HDFS文件操作（增删改读）

<span style="background-color: rgb(255, 255, 255);">1、 代码                                                                                                                                         </span>

package com.ctrip.bi.uss.util;import java.io.BufferedReader;import java.io.IOException;import java.io.InputStreamReader;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.log4j.Logger;import com.ctrip.bi.uss.mgr.LogMgr;public class HdfsUtils {private static Logger log = LogMgr.getUss_log();public static void main(String[] args) throws Exception {copyHDFS2Local("hdfs://ns/tmp/data/index/2014-09-05/","/opt/ctrip/web/work/data/ws/");}// 上传文件到hdfspublic static void uploadLocalFileHDFS(String s, String d)throws IOException {Configuration config = new Configuration();FileSystem hdfs = FileSystem.get(config);Path src = new Path(s);Path dst = new Path(d);hdfs.copyFromLocalFile(src, dst);hdfs.close();}// 拷贝整个hdfs下的文件夹到本地public static boolean copyHDFS2Local(String hdfsFilePath,String localFilePath) throws Exception {Configuration conf = new Configuration();FileSystem fs = FileSystem.get(conf);Path localPath = new Path(localFilePath);Path hdfsPath = new Path(hdfsFilePath);try {fs.copyToLocalFile(hdfsPath, localPath);return true;} catch (Exception e) {log.error(e.getMessage() + " " + e.getCause());return false;}}// 在hdfs中创建文件public static void createNewHDFSFile(String toCreateFilePath, String content)throws IOException {Configuration config = new Configuration();FileSystem hdfs = FileSystem.get(config);FSDataOutputStream os = hdfs.create(new Path(toCreateFilePath));os.write(content.getBytes("UTF-8"));os.close();hdfs.close();}// 删除hdfs中文件public static boolean deleteHDFSFile(String dst) throws IOException {Configuration config = new Configuration();FileSystem hdfs = FileSystem.get(config);Path path = new Path(dst);boolean isDeleted = hdfs.delete(path);hdfs.close();return isDeleted;}// 读取hdfs文件内容public static byte[] readHDFSFile(String dst) throws Exception {<span style="white-space:pre"></span>Configuration conf = new Configuration();<span style="white-space:pre"></span>FileSystem fs = FileSystem.get(conf);<span style="white-space:pre"></span>// check if the file exists<span style="white-space:pre"></span>Path path = new Path(dst);<span style="white-space:pre"></span>if (fs.exists(path)) {<span style="white-space:pre"></span>FSDataInputStream is = fs.open(path);<span style="white-space:pre"></span>// get the file info to create the buffer<span style="white-space:pre"></span>FileStatus stat = fs.getFileStatus(path);<span style="white-space:pre"></span>// create the buffer<span style="white-space:pre"></span>byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat<span style="white-space:pre"></span>.getLen()))];<span style="white-space:pre"></span>is.readFully(0, buffer);<span style="white-space:pre"></span>is.close();<span style="white-space:pre"></span>fs.close();

<span style="white-space:pre"></span>return buffer;<span style="white-space:pre"></span>} else {<span style="white-space:pre"></span>throw new Exception("the file is not found .");<span style="white-space:pre"></span>}<span style="white-space:pre"></span>}// 新建目录public static void mkdir(String dir) throws IOException {Configuration conf = new Configuration();FileSystem fs = FileSystem.get(conf);fs.mkdirs(new Path(dir));fs.close();}// 删除目录public static void deleteDir(String dir) throws IOException {Configuration conf = new Configuration();FileSystem fs = FileSystem.get(conf);fs.delete(new Path(dir));fs.close();}// 列出所有的文件public static void listAll(String src, String local) throws IOException {Configuration conf = new Configuration();FileSystem fs = FileSystem.get(conf);FileStatus[] stats = fs.listStatus(new Path(src));for (int i = 0; i < stats.length; ++i) {if (stats[i].isFile()) {System.out.println(stats[i].getPath().toString());} else if (stats[i].isDirectory()) {System.out.println(stats[i].getPath().toString());} else if (stats[i].isSymlink()) {System.out.println(stats[i].getPath().toString());}}fs.close();}}

2、配置文件 core-site.xml （此文件需要放在类的根目录下）

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><configuration><property><name>fs.defaultFS</name><value>hdfs://ns</value></property><property><name>fs.file.impl</name><value>org.apache.hadoop.fs.LocalFileSystem</value><description>The FileSystem for file: uris.</description></property><property><name>fs.hdfs.impl</name><value>org.apache.hadoop.hdfs.DistributedFileSystem</value><description>The FileSystem for hdfs: uris.</description></property><property><name>dfs.nameservices</name><value>ns</value></property><property><name>dfs.ha.namenodes.ns</name><value>SVR2368HP360,SVR2369HP360</value></property><property><name>dfs.namenode.rpc-address.ns.SVR2368HP360</name><value>SVR2368HP360.hadoop.test.sh.ctriptravel.com:54310</value></property><property><name>dfs.namenode.rpc-address.ns.SVR2369HP360</name><value>SVR2369HP360.hadoop.test.sh.ctriptravel.com:54310</value></property><property><name>dfs.client.failover.proxy.provider.ns</name><value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value></property></configuration>

以上的配置信息每台机子各异，因此需要相应改动而不能直接copy

3、 必须的jar包

commons-cli-1.2.jar

commons-collections-3.1.jar

commons-configuration-1.6.jar

commons-io-2.1.jar

commons-lang-2.5.jar

commons-logging-1.1.1.jar

guava-11.0.2.jar

hadoop-auth-2.0.0-cdh4.2.1.jar

hadoop-common-2.0.0-cdh4.6.0.jar

hadoop-core.jar

hadoop-hdfs-2.0.0-cdh4.6.0.jar

log4j-1.2.17.jar

protobuf-java-2.4.0a.jar

slf4j-api-1.6.1.jar

slf4j-log4j12-1.6.1.jar