SequenceFileInputFormat用法

SequenceFileInputFormat只能处理SequenceFile类型的文件。

代码：

package inputformat;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;//用之前SequenceFile类型的文件作为处理数据，用那个for循环生成的数据，那个数据指定的类型是<LongWritable,Text>//SequenceFileInputFormat只能处理SequenceFile类型的数据public class SequenceFileInputFormatTest {public static class MyMapper extendsMapper<LongWritable, Text, Text, LongWritable> {final Text k2 = new Text();final LongWritable v2 = new LongWritable();protected void map(LongWritable key, Text value,Mapper<LongWritable, Text, Text, LongWritable>.Context context)throws InterruptedException, IOException {final String line = value.toString();final String[] splited = line.split("\\s");for (String word : splited) {k2.set(word);v2.set(1);context.write(k2, v2);}}}public static class MyReducer extendsReducer<Text, LongWritable, Text, LongWritable> {LongWritable v3 = new LongWritable();protected void reduce(Text k2, Iterable<LongWritable> v2s,Reducer<Text, LongWritable, Text, LongWritable>.Context context)throws IOException, InterruptedException {long count = 0L;for (LongWritable v2 : v2s) {count += v2.get();}v3.set(count);context.write(k2, v3);}}public static void main(String[] args) throws Exception {final Configuration conf = new Configuration();final Job job = Job.getInstance(conf, SequenceFileInputFormatTest.class.getSimpleName());// 1.1FileInputFormat.setInputPaths(job,"hdfs://192.168.1.10:9000/sf1");//这里改了一下，把TextInputFormat改成了SequenceFileInputFormatjob.setInputFormatClass(SequenceFileInputFormat.class);// 1.2job.setMapperClass(MyMapper.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(LongWritable.class);// 1.3 默认只有一个分区job.setPartitionerClass(HashPartitioner.class);job.setNumReduceTasks(1);// 1.4省略不写// 1.5省略不写// 2.2job.setReducerClass(MyReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(LongWritable.class);// 2.3FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.1.10:9000/out1"));job.setOutputFormatClass(TextOutputFormat.class);// 执行打成jar包的程序时，必须调用下面的方法job.setJarByClass(SequenceFileInputFormatTest.class);job.waitForCompletion(true);}}

生成SequenceFile类型的文件，供上述SequenceFileInputFormat使用，作为输入数据：

package sequenceFile;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.SequenceFile;import org.apache.hadoop.io.Text;import org.apache.zookeeper.common.IOUtils;//for循环读写操作演示public class Forduxie {public static void main(String args[]) throws Exception {final Path path = new Path("/sf1");Configuration conf = new Configuration();final FileSystem fs = FileSystem.get(new URI("hdfs://192.168.1.10:9000/"), conf);@SuppressWarnings("deprecation")final SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf,path, LongWritable.class,Text.class);for (int i = 0; i < 10; i++) {writer.append(new LongWritable(i), new Text(i+"=_="));}IOUtils.closeStream(writer);@SuppressWarnings({ "deprecation" })final SequenceFile.Reader reader = new SequenceFile.Reader(fs, path,conf);LongWritable key = new LongWritable();Text val = new Text();while (reader.next(key, val)) {System.out.println(key.get() + "\t" + val.toString());}IOUtils.closeStream(reader);}}

如果创建的是Maven项目，需要在pom包里添加：

<span style="white-space:pre"></span><dependency><groupId>commons-io</groupId><artifactId>commons-io</artifactId><version>2.4</version></dependency>