hadoop学习5

利用SequenceFile对文件进行读写
1.文件的写操作

import java.io.IOException;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.SequenceFile;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.SequenceFile.CompressionType;public class SequenceFileWriteDemo {    private static String[] myValue = {"hello world","bye world","hello hadoop","bye world"};    public static void main(String[] args) throws IOException{        String uri = "hdfs://node1:9000/afan/output";        Configuration conf = new Configuration();        FileSystem fs = FileSystem.get(URI.create(uri),conf);        Path path = new Path(uri);        IntWritable key = new IntWritable();        Text value = new Text();        SequenceFile.Writer writer = null;        try{            //对文件不进行压缩//          writer = SequenceFile.createWriter(fs,conf,path,key.getClass(),value.getClass());//                      //对文件进行压缩            writer = SequenceFile.createWriter(fs,conf,path,key.getClass(),value.getClass()                    ,CompressionType.BLOCK);            for(int i = 0;i<50;i++){                key.set(50-i);                value.set(myValue[i%myValue.length]);                writer.append(key,value);            }        }finally{            IOUtils.closeStream(writer);        }    }}

2.文件的读操作

import java.io.IOException;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.apache.hadoop.io.SequenceFile;import org.apache.hadoop.io.Writable;import org.apache.hadoop.util.ReflectionUtils;public class SequenceFileReadFile {    public static void main(String[] args) throws IOException{        String uri = "hdfs://node1:9000/afan/output";        Configuration conf = new Configuration();        FileSystem fs = FileSystem.get(URI.create(uri),conf);        Path path = new Path(uri);        SequenceFile.Reader reader = null;        try{            reader = new SequenceFile.Reader(fs,path, conf);            Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);            Writable value =(Writable) ReflectionUtils.newInstance(reader.getValueClass(),conf);            while(reader.next(key, value)){                System.out.println(key + "---"+ value);            }        }finally{            IOUtils.closeStream(reader);        }    }}