hadoop入门demo

来源：互联网发布：域名注册godaddy 编辑：程序博客网时间：2024/06/05 17:29

1、测试类

package com.wl.hdfs.test1;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;

public class Test1 {

    public static void main(String[] args) throws Exception {
//         MkDir();
        // delDir();
        // delFile();
        upload();
//        upload2();
    }

    /**
     *
     * TODO:方法描述
     *
     * @version 2016年7月7日上午10:48:00
     * @author wuliu
     * @throws Exception
     */
    private static void MkDir() throws Exception {
        FileSystem fs = FileSystem.get(new URI("hdfs://123.206.16.225:9000"), new Configuration());
        fs.mkdirs(new Path("/upload1/test"));
        fs.close();
    }

    /**
     * 删除test目录
     *
     * @version 2016年7月7日上午10:47:46
     * @author wuliu
     * @throws Exception
     */
    private static void delDir() throws Exception {
        FileSystem fs = FileSystem.get(new URI("hdfs://123.206.16.225:9000"), new Configuration());
        fs.delete(new Path("/upload1/test"), true);
        fs.close();
    }

    private static void delFile() throws Exception {
        FileSystem fs = FileSystem.get(new URI("hdfs://123.206.16.225:9000"), new Configuration());
        fs.delete(new Path("/upload/svn1.txt"), true);
        fs.close();
    }
    //有问题
    private static void upload() throws Exception {
        FileSystem fs = FileSystem.get(new URI("hdfs://123.206.16.225:9000"), new Configuration());
        FSDataOutputStream fsDataOutputStream = fs.create(new Path("/upload/svn1.txt"));
        FileInputStream in = new FileInputStream(new File("G://svn.txt"));
        IOUtils.copyBytes(in, fsDataOutputStream, 2048, true);
        fs.close();
    }
   //有问题
    private static void upload2() throws Exception {
        String localFileSrc = "F://svn.txt";
        String dst = "hdfs://123.206.16.225:9000/user/svn.txt";
        InputStream in = new BufferedInputStream(new FileInputStream(localFileSrc));
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(dst),configuration);
        OutputStream out = fs.create(new Path(dst), new Progressable() {
            public void progress() {
                System.out.println(".");
            }
        });
        IOUtils.copyBytes(in, out, 2048,true);
        System.out.println("success");
    }

}

2、pom.xml配置

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.jkxy</groupId>
   <artifactId>hdfstest</artifactId>
   <version>0.0.1-SNAPSHOT</version>
   <properties>
       <junit.version>4.7</junit.version>
       <hadoop.version>2.7.1</hadoop.version>
   </properties>
   <dependencies>
       <dependency>
           <groupId>junit</groupId>
           <artifactId>junit</artifactId>
           <version>${junit.version}</version>
       </dependency>
       <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-common</artifactId>
           <version>${hadoop.version}</version>
       </dependency>
       <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-hdfs</artifactId>
           <version>${hadoop.version}</version>
       </dependency>

       <dependency>
           <groupId>jdk.tools</groupId>
           <artifactId>jdk.tools</artifactId>
           <version>1.7</version>
           <scope>system</scope>
           <systemPath>D:/Software/Install/Java/jdk1.7.0_80/lib/tools.jar</systemPath>
       </dependency>

   </dependencies>
</project>

0 0