Hadoop javaAPI

来源：互联网发布：英语软件有哪些编辑：程序博客网时间：2024/05/20 11:23

1、读

package com.sdnware.start01.hadoop;import java.io.ByteArrayOutputStream;import java.io.IOException;import java.io.InputStream;import java.net.URI;import java.net.URISyntaxException;import java.util.Arrays;import java.util.List;import java.util.function.Consumer;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;public class HDFSRead {    public static void main(String[] args) {        read1();    }    public static void read0(){        System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");        System.setProperty("HADOOP_USER_NAME","root");//制定上传的用户        Configuration conf = new Configuration();        try {            FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);            InputStream in = hdfs.open(new Path("/user/wordcount/My Father0.txt"));            ByteArrayOutputStream out = new ByteArrayOutputStream();            IOUtils.copyBytes(in, out, 4096,true);            String str = out.toString();            System.out.println(str);        } catch (IOException e) {            e.printStackTrace();        } catch (URISyntaxException e) {            e.printStackTrace();        }    }    public static void read1(){        System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");        System.setProperty("HADOOP_USER_NAME","root");//制定上传的用户        Configuration conf = new Configuration();        try {            final FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);            Path path = new Path("/user/wordcount");            if(hdfs.exists(path)){                List<FileStatus> listStatus = Arrays.asList(hdfs.listStatus(path));                listStatus.forEach(new Consumer<FileStatus>() {                    public void accept(FileStatus t) {                        if(t.isFile()){                            FSDataInputStream in = null;                            try {                                Path path2 = t.getPath();                                in = hdfs.open(path2);                                System.out.println("Read File："+ path2.getName());                                ByteArrayOutputStream out = new ByteArrayOutputStream();                                IOUtils.copyBytes(in, out, 4096,true);                                String str = out.toString();                                System.out.println(str);                            } catch (IOException e) {                                e.printStackTrace();                            }finally{                                IOUtils.closeStream(in);                            }                        }                    }                });            }        } catch (IOException e) {            e.printStackTrace();        } catch (URISyntaxException e) {            e.printStackTrace();        }    }}

2、写

package com.sdnware.start01.hadoop;import java.io.IOException;import java.net.URI;import java.net.URISyntaxException;import org.apache.commons.io.IOUtils;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;public class HDFSWrite {    public static void main(String[] args) {        HDFSWrite writer = new HDFSWrite();        writer.writer1();    }    public void writer0() {        FSDataOutputStream out = null;        FSDataInputStream in = null;        try {            System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");            Configuration conf = new Configuration();            FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);            FileSystem local = FileSystem.getLocal(conf);            Path inputDir = new Path("D:/testfile/"); // 获取本地待被上传的文件                                                        // 会把该文件夹下的内容都上传            Path hdfsFile = new Path("/user/wordcount"); // 获取hdfs上需要获取文件的地址            hdfs.mkdirs(hdfsFile); // 在hdfs上新建该文件夹            FileStatus[] inputFiles = local.listStatus(inputDir); // 获取本地待上传文件夹的信息            for (int i = 0; i < inputFiles.length; i++) { // i循环本地文件夹内的文件个数                System.out.println(inputFiles[i].getPath().getName());                in = local.open(inputFiles[i].getPath()); // 获取输入流                out = hdfs.create(new Path("/user/wordcount/" + inputFiles[i].getPath().getName())); // 输出流位置获取                byte buffer[] = new byte[256];                int bytesRead = 0;                while ((bytesRead = in.read(buffer)) > 0) { // 每次读取buffer大小的部分                    out.write(buffer, 0, bytesRead); // 每次写入buffer中bytesRead大小的部分                }            }        } catch (IOException e) {            e.printStackTrace();        } catch (URISyntaxException e) {            e.printStackTrace();        }finally{            IOUtils.closeQuietly(in);            IOUtils.closeQuietly(out);        }    }    public void writer1() {        FSDataOutputStream out = null;        FSDataInputStream in = null;        try {            System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");            System.setProperty("HADOOP_USER_NAME","root");//制定上传的用户            Configuration conf = new Configuration();            FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);            FileSystem local = FileSystem.getLocal(conf);            Path inputDir = new Path("D:/testfile/"); // 获取本地待被上传的文件                                                        // 会把该文件夹下的内容都上传            Path hdfsFile = new Path("/user/wordcount"); // 获取hdfs上需要获取文件的地址            hdfs.mkdirs(hdfsFile); // 在hdfs上新建该文件夹            FileStatus[] inputFiles = local.listStatus(inputDir); // 获取本地待上传文件夹的信息            for (int i = 0; i < inputFiles.length; i++) { // i循环本地文件夹内的文件个数                System.out.println(inputFiles[i].getPath().getName());                in = local.open(inputFiles[i].getPath()); // 获取输入流                out = hdfs.create(new Path("/user/wordcount/" + inputFiles[i].getPath().getName())); // 输出流位置获取                org.apache.hadoop.io.IOUtils.copyBytes(in, out, 4096,true);            }        } catch (IOException e) {            e.printStackTrace();        } catch (URISyntaxException e) {            e.printStackTrace();        }finally{            org.apache.hadoop.io.IOUtils.closeStream(in);            org.apache.hadoop.io.IOUtils.closeStream(out);        }    }}

3、删除

package com.sdnware.start01.hadoop;import java.io.IOException;import java.net.URI;import java.net.URISyntaxException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;public class HDFSDelete {    public static void main(String[] args) {        delete1();    }    public static void delete0(){//删除文件        System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");        System.setProperty("HADOOP_USER_NAME","root");//制定上传的用户        Configuration conf = new Configuration();        try {            FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);            Path hdfsFile = new Path("/user/wordcount/My Father.txt");            hdfs.delete(hdfsFile, false);        } catch (IOException e) {            e.printStackTrace();        } catch (URISyntaxException e) {            e.printStackTrace();        }    }    public static void delete1(){//删除目录        System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");        System.setProperty("HADOOP_USER_NAME","root");//制定上传的用户        Configuration conf = new Configuration();        try {            FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);            Path hdfsFile = new Path("/usr");            hdfs.delete(hdfsFile, true);        } catch (IOException e) {            e.printStackTrace();        } catch (URISyntaxException e) {            e.printStackTrace();        }    }}

0 0