HDFS之SequenceFile的读写操作

本文代码参考hadoop权威指南里第四章hadoop的i/o操作

SequenceFile写操作：

import java.io.IOException;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.SequenceFile;import org.apache.hadoop.io.Text;//sequenceFile追加操作bytepublic class SequenceFileWriteDemo {private static final String[] data = {"kcbhducdi","jhcui","lashdcuohco"};public static void main(String args[]){String uri = args[0];Configuration conf = new Configuration();FileSystem fs = null;;try {fs = FileSystem.get(URI.create(uri), conf);} catch (IOException e1) {// TODO Auto-generated catch blocke1.printStackTrace();}Path path = new Path(uri);IntWritable key = new IntWritable();Text value = new Text();SequenceFile.Writer write = null;try {write = SequenceFile.createWriter(fs, conf, path,key.getClass(),value.getClass());for(int i=0;i< 100;i++){key.set(100-i);value.set(data[i%data.length]);System.out.printf("[%d]\t%s\t%s\n",write.getLength(),key,value);write.append(key, value);} }catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}finally{IOUtils.closeStream(write);}}}

输出：

[128]100kcbhducdi[158]99jhcui[184]98lashdcuohco[216]97kcbhducdi[246]96jhcui[272]95lashdcuohco[304]94kcbhducdi[334]93jhcui[360]92lashdcuohco[392]91kcbhducdi[422]90jhcui[448]89lashdcuohco[480]88kcbhducdi[510]87jhcui[536]86lashdcuohco[568]85kcbhducdi[598]84jhcui[624]83lashdcuohco[656]82kcbhducdi[686]81jhcui[712]80lashdcuohco[744]79kcbhducdi[774]78jhcui[800]77lashdcuohco[832]76kcbhducdi[862]75jhcui[888]74lashdcuohco[920]73kcbhducdi[950]72jhcui[976]71lashdcuohco[1008]70kcbhducdi[1038]69jhcui[1064]68lashdcuohco[1096]67kcbhducdi[1126]66jhcui[1152]65lashdcuohco[1184]64kcbhducdi[1214]63jhcui[1240]62lashdcuohco[1272]61kcbhducdi[1302]60jhcui[1328]59lashdcuohco[1360]58kcbhducdi[1390]57jhcui[1416]56lashdcuohco[1448]55kcbhducdi[1478]54jhcui[1504]53lashdcuohco[1536]52kcbhducdi[1566]51jhcui[1592]50lashdcuohco[1624]49kcbhducdi[1654]48jhcui[1680]47lashdcuohco[1712]46kcbhducdi[1742]45jhcui[1768]44lashdcuohco[1800]43kcbhducdi[1830]42jhcui[1856]41lashdcuohco[1888]40kcbhducdi[1918]39jhcui[1944]38lashdcuohco[1976]37kcbhducdi[2006]36jhcui[2052]35lashdcuohco[2084]34kcbhducdi[2114]33jhcui[2140]32lashdcuohco[2172]31kcbhducdi[2202]30jhcui[2228]29lashdcuohco[2260]28kcbhducdi[2290]27jhcui[2316]26lashdcuohco[2348]25kcbhducdi[2378]24jhcui[2404]23lashdcuohco[2436]22kcbhducdi[2466]21jhcui[2492]20lashdcuohco[2524]19kcbhducdi[2554]18jhcui[2580]17lashdcuohco[2612]16kcbhducdi[2642]15jhcui[2668]14lashdcuohco[2700]13kcbhducdi[2730]12jhcui[2756]11lashdcuohco[2788]10kcbhducdi[2818]9jhcui[2844]8lashdcuohco[2876]7kcbhducdi[2906]6jhcui[2932]5lashdcuohco[2964]4kcbhducdi[2994]3jhcui[3020]2lashdcuohco[3052]1kcbhducdi

SequenceFile读操作：

import java.io.IOException;import java.net.URI;import org.apache.commons.io.IOUtils;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.SequenceFile;import org.apache.hadoop.io.Writable;import org.apache.hadoop.util.ReflectionUtils;//sequenceFile读取操作public class SequenceFileReadDemo {public static void main(String args[]) throws IOException{String uri = args[0];Configuration conf = new Configuration();FileSystem fs = FileSystem.get(URI.create(uri),conf);Path path = new Path(uri);SequenceFile.Reader reader = null;try {reader = new SequenceFile.Reader(fs, path, conf);Writable key = (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf);Writable value = (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf);long position = reader.getPosition();while(reader.next(key,value)){String syncSeen = reader.syncSeen() ? "*" : "" ;//set the reader markSystem.out.printf("[%s%s]\t%s\t%s\n",position,syncSeen,key,value);position = reader.getPosition();}} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}finally{reader.close();//IOUtils.closeStream(reader);;}}}

输出：

[128]100kcbhducdi[158]99jhcui[184]98lashdcuohco[216]97kcbhducdi[246]96jhcui[272]95lashdcuohco[304]94kcbhducdi[334]93jhcui[360]92lashdcuohco[392]91kcbhducdi[422]90jhcui[448]89lashdcuohco[480]88kcbhducdi[510]87jhcui[536]86lashdcuohco[568]85kcbhducdi[598]84jhcui[624]83lashdcuohco[656]82kcbhducdi[686]81jhcui[712]80lashdcuohco[744]79kcbhducdi[774]78jhcui[800]77lashdcuohco[832]76kcbhducdi[862]75jhcui[888]74lashdcuohco[920]73kcbhducdi[950]72jhcui[976]71lashdcuohco[1008]70kcbhducdi[1038]69jhcui[1064]68lashdcuohco[1096]67kcbhducdi[1126]66jhcui[1152]65lashdcuohco[1184]64kcbhducdi[1214]63jhcui[1240]62lashdcuohco[1272]61kcbhducdi[1302]60jhcui[1328]59lashdcuohco[1360]58kcbhducdi[1390]57jhcui[1416]56lashdcuohco[1448]55kcbhducdi[1478]54jhcui[1504]53lashdcuohco[1536]52kcbhducdi[1566]51jhcui[1592]50lashdcuohco[1624]49kcbhducdi[1654]48jhcui[1680]47lashdcuohco[1712]46kcbhducdi[1742]45jhcui[1768]44lashdcuohco[1800]43kcbhducdi[1830]42jhcui[1856]41lashdcuohco[1888]40kcbhducdi[1918]39jhcui[1944]38lashdcuohco[1976]37kcbhducdi[2006*]36jhcui[2052]35lashdcuohco[2084]34kcbhducdi[2114]33jhcui[2140]32lashdcuohco[2172]31kcbhducdi[2202]30jhcui[2228]29lashdcuohco[2260]28kcbhducdi[2290]27jhcui[2316]26lashdcuohco[2348]25kcbhducdi[2378]24jhcui[2404]23lashdcuohco[2436]22kcbhducdi[2466]21jhcui[2492]20lashdcuohco[2524]19kcbhducdi[2554]18jhcui[2580]17lashdcuohco[2612]16kcbhducdi[2642]15jhcui[2668]14lashdcuohco[2700]13kcbhducdi[2730]12jhcui[2756]11lashdcuohco[2788]10kcbhducdi[2818]9jhcui[2844]8lashdcuohco[2876]7kcbhducdi[2906]6jhcui[2932]5lashdcuohco[2964]4kcbhducdi[2994]3jhcui[3020]2lashdcuohco[3052]1kcbhducdi