《小白兔到大黑牛》第十三篇maven中Hadoop中类的练习

pom.xml配置如下[html] view plain copy    <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"      xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">      <modelVersion>4.0.0</modelVersion>      <parent>      <artifactId>storage</artifactId>      <groupId>platform.edu.storage</groupId>      <version>0.0.1-SNAPSHOT</version>      </parent>      <artifactId>explorer-impl</artifactId>      <name>${project.artifactId}</name>      <packaging>jar</packaging>      <description>Hadoop HDFS explorer实现</description>      <dependencies>      <!-- hadoop base before -->      <dependency>      <groupId>org.apache.hadoop</groupId>      <artifactId>hadoop-client</artifactId>      <version>2.2.0</version>      </dependency>      <!-- hadood base end -->      </dependencies>  HDFSServer.Java代码如下-----------------------------------------------------------[java] view plain copy    package platform.edu.explorer.hdfs.server;      import java.io.IOException;      import java.net.URI;      import org.apache.hadoop.conf.Configuration;      import org.apache.hadoop.fs.FileSystem;      import org.apache.hadoop.mapred.JobConf;      /**      * HDFS服务器      *       * @author hjn      * @version 1.0 2013-11-20      */      public class HDFSServer {      private static Configuration configuration;      private static FileSystem fileSystem;      private static final String HDFS_URL = "hdfs://192.168.1.210:8020";      /**      * HDFS服务器读取初始化      */      private static void init() {      try {      configuration= new JobConf(HDFSServer.class);      fileSystem = FileSystem.get(URI.create(HDFS_URL), configuration);      } catch (IOException e) {      System.out.println("读取服务器失败");      e.printStackTrace();      }      }      public static FileSystem getFileSystem(){      if(fileSystem==null){      init();      }      return fileSystem;      }      }      --------------------------------------------------------------------------------------------------------------------------------------------------      HDFS.java代码如下      --------------------------      package platform.edu.explorer.hdfs;      import java.io.File;      import java.io.FileInputStream;      import java.io.FileNotFoundException;      import java.io.FileOutputStream;      import java.io.IOException;      import java.io.InputStream;      import java.io.OutputStream;      import java.util.ArrayList;      import java.util.List;      import org.apache.commons.compress.utils.IOUtils;      import org.apache.hadoop.fs.FSDataInputStream;      import org.apache.hadoop.fs.FSDataOutputStream;      import org.apache.hadoop.fs.FileStatus;      import org.apache.hadoop.fs.FileSystem;      import org.apache.hadoop.fs.Path;      import platform.edu.explorer.HadoopFileSystem;      import platform.edu.explorer.hdfs.server.HDFSServer;      /**      * HDFS(分布式文件存储系统操作类).      *       * @author hjn       * @version 1.0 2013-11-22      */      public class HDFS implements HadoopFileSystem {      /**      * 文件系统类      */      private FileSystem fileSystem;      /**      * 无参数构造方法      */      public HDFS() {         init();      }      /**      * 初始化      */      private void init() {         fileSystem = HDFSServer.getFileSystem();      }      /**      * 获取HDFS指定目录下文件状态列表      *       * @param dirPath指定目录路径      * @return fileStatusList      * @throws FileNotFoundException      * @throws IOException      */      public FileStatus[] getFileStatus(Path path) throws FileNotFoundException,      IOException {         FileStatus[] fileStatusList = fileSystem.listStatus(path);         return fileStatusList;      }      /**      * 获取指定目录列表路径      *       * @param dirPath      */      public List<String> dir(String dirPath) throws IOException {        List<String> fileList = null;        Path path = new Path(dirPath);        if (fileSystem.exists(path)) {          fileList = new ArrayList<String>();          FileStatus[] list = this.getFileStatus(path);        for (FileStatus fileStatus : list) {          fileList.add(fileStatus.getPath().toString());        }        } else {           System.out.println("目录不存在");        }        return fileList;      }      /**      * 获取文件      *       * @param filePath      * @return      * @throws IOException      */      public InputStream getFile(String filePath) throws IOException {      Path path = new Path(filePath);      return fileSystem.open(path);      }      /**      * 更改HDSF文件名称      *       * @param fileOldName      * @param fileNewName      * @return boolean:是否更名字成功      * @throws IOException      */      public boolean rename(String src, String dst) throws IOException {         Path srcPath = new Path(src);         if (fileSystem.exists(srcPath)) {           Path dstPath = new Path(dst);           return fileSystem.rename(srcPath, dstPath);         }         System.out.println("原文件不存在");         return false;      }      /**      * 创建HDFS目录      *       * @param dir      */      public boolean createDir(String dir) throws IOException {         Path path = new Path(dir);         if (fileSystem.exists(path)) {           System.out.println("此目录已经存在不需要再创建");         return true;         }         return fileSystem.mkdirs(path);      }      /**      * 上传本地文件到HDFS（注意是服务器本地硬盘，非客户端硬盘)）      *       * @return      *       * @throws IOException      */      @Override      public void uploadLocalFile(String localFileSrc, String HDFSFileDst)                 throws IOException {        Path src = new Path(localFileSrc);        Path dst = new Path(HDFSFileDst);        fileSystem.copyFromLocalFile(src, dst);      }      /**      * 批量上传本地文件到HDFS      * @param localFileSrcs本地文件列表      * @param HDFSFileDst      * @throws IOException      */          public void uploadLocalFile(String[] localFileSrcs,String HDFSFileDst) throws IOException{          Path dstPath=new Path(HDFSFileDst);          Path[] paths=new Path[localFileSrcs.length];          for (int i=0;i<localFileSrcs.length;i++) {      paths[i]=new Path(localFileSrcs[i]);      }          fileSystem.copyFromLocalFile(false, false, paths,dstPath);          }      /**      * 从HDFS下载文件到本地(注意是服务器本地硬盘，非浏览器客户端硬盘)      *       * @param HDFSFilePath      * @param localFilePath      * @throws IOException      */      public void downFileToLocal(String HDFSFilePath, String localFilePath)      throws IOException {        Path dstPath = new Path(HDFSFilePath);        FSDataInputStream in = fileSystem.open(dstPath);        OutputStream out = new FileOutputStream(new File(localFilePath));        IOUtils.copy(in, out);      }      /**      * HDFS文件是否存在      *       * @param filePath      * @return boolean:是否存在      * @throws IOException      */      public boolean exists(String filePath) throws IOException {        Path path = new Path(filePath);        return fileSystem.exists(path);      }      /**      * 根据路径删除文件或文件夹      *       * @param filePath      * @return      * @throws IOException      */      @SuppressWarnings("deprecation")      public boolean deleteFile(String filePath) throws IOException {        if (this.exists(filePath)) {          Path path = new Path(filePath);          fileSystem.delete(path);          return true;        }        System.out.println("文件不存在");        return false;      }      /**      * 剪切本地文件到HDFS(注意为服务器本地文件);      *       * @param src本地路径      * @param dst分布式存储路径      * @throws IOException      */      public void moveFromLocalFile(String localSrc, String HDFSDst) throws IOException {         Path srcPath = new Path(localSrc);         Path dstPath = new Path(HDFSDst);         fileSystem.moveFromLocalFile(srcPath, dstPath);      }      /**      * HDFS文件之间的复制      * @param src源文件路径      * @param dst要复制后复制文件的路径      * @throws IOException      */      public void copyHDFSFile(String src,String dst) throws IOException{         Path srcPath=new Path(src);         Path dstPath=new Path(dst);         InputStream in=fileSystem.open(srcPath);         OutputStream out=fileSystem.create(dstPath);         IOUtils.copy(in, out);      }      /**      * HDFS中移动文件      * @param src源文件路径      * @param dst要移动后的路径      * @throws IOException      */      @SuppressWarnings("deprecation")      public void moveHDFSFile(String src,String dst) throws IOException{         Path srcPath=new Path(src);         Path dstPath=new Path(dst);         InputStream in=fileSystem.open(srcPath);         OutputStream out=fileSystem.create(dstPath);         IOUtils.copy(in, out);         fileSystem.delete(srcPath);      }      /**      * 剪切HDFS文件到本地      * @param HDFSSrc      * @param localDst      * @throws IOException      */          public void moveToLocalFile(String HDFSSrc,String localDst) throws IOException{          Path srcPath = new Path(HDFSSrc);           Path dstPath = new Path(localDst);          fileSystem.moveToLocalFile(srcPath, dstPath);          }      /**      * HDFS创建文件      *       * @param in输入流      * @param dst分布式存储路径      * @throws IOException      */      public void create(InputStream in, String dst) throws IOException {         Path dstPath = new Path(dst);         FSDataOutputStream out = fileSystem.create(dstPath);         IOUtils.copy(in, out);      }      /**      * 在HDFS创建文件      *       * @param file      * @param dst分布式存储路径      * @throws IOException      */      public void create(File file, String dst) throws IOException {         InputStream in = new FileInputStream(file);         this.create(in, dst);      }      /**      * 在HDFS创建文件      *       * @param src本地文件路径      * @param dst分布式存储路径      * @throws IOException      */      public void create(String src, String dst) throws IOException {         File file = new File(src);         this.create(file, dst);      }      /**      * 获取FileSystem对象      *       * @return      */      public FileSystem getFileSystem() {         return fileSystem;      }      /**      * 关闭HDFS      *       * @throws IOException      */      public void close() throws IOException {         fileSystem.close();      }      public boolean isDir(String src) throws IOException {         Path path = new Path(src);         return fileSystem.isDirectory(path);      }      public static void main(String[] args) throws IOException {            HDFS hdfs =           new HDFS();          // hdfs.uploadLocalFile("D:/picture", "/test"); //            System.          out.println(hdfs.dir("/"));          // hdfs.create("D:/picture/mypicture/POP海报2590.jpg","/test/picture/mypicture/POP海报2590.jpg");            System.          out.println(hdfs.dir("/test/picture/mypicture"));          //hdfs.uploadLocalFile(new String[]{"E:/input","E:/output"}, "/");          //hdfs.rename("/input", "/debug_in");          System.          out.println(hdfs.dir("/debug_out"));          //hdfs.deleteFile("/output");          // hdfs.moveFromLocalFile("E:/test.jpg", "/test/picture/POP海报2590.jpg");          /*          * System.out.println(hdfs.dir("/test"));          * System.out.println(hdfs.exists("/test/picture/mypicture")); //          * hdfs.delete("/test/picture/mypicture");          * System.out.println(hdfs.dir("/test/picture"));          */          hdfs.close();      }      }