RCFile读写操作

http://smallboby.iteye.com/blog/1592531

读取RCfile

Job job = new Job();      job.setJarByClass(类.class);      //设定输入文件为RcFile格式      job.setInputFormatClass(RCFileInputFormat.class);         //普通输出      job.setOutputFormatClass(TextOutputFormat.class);       //设置输入路径      RCFileInputFormat.addInputPath(job, new Path(srcpath));      //MultipleInputs.addInputPath(job, new Path(srcpath), RCFileInputFormat.class);       // 输出      TextOutputFormat.setOutputPath(job, new Path(respath));      // 输出key格式      job.setOutputKeyClass(Text.class);        //输出value格式      job.setOutputValueClass(NullWritable.class);        //设置mapper类       job.setMapperClass(ReadTestMapper.class);              //这里没设置reduce，reduce的操作就是读Text类型文件，因为mapper已经给转换了。                                code = (job.waitForCompletion(true)) ? 0 : 1;              // mapper 类            pulic class ReadTestMapper extends                 Mapper<LongWritable, BytesRefArrayWritable, Text, NullWritable> {                            @Override              protected void map(LongWritable key, BytesRefArrayWritable value, Context context)                     throws IOException, InterruptedException {                  // TODO Auto-generated method stub                  Text txt = new Text();               //因为RcFile行存储和列存储，所以每次进来的一行数据，Value是个列簇，遍历，输出。                  StringBuffer sb = new StringBuffer();                  for (int i = 0; i < value.size(); i++) {                      BytesRefWritable v = value.get(i);                      txt.set(v.getData(), v.getStart(), v.getLength());                      if(i==value.size()-1){                          sb.append(txt.toString());                      }else{                          sb.append(txt.toString()+"\t");                      }                  }                  context.write(new Text(sb.toString()),NullWritable.get());              }       }

写RCFile

Job job = new Job();      Configuration conf = job.getConfiguration();  //设置每行的列簇数      RCFileOutputFormat.setColumnNumber(conf, 4);      job.setJarByClass(类.class);      FileInputFormat.setInputPaths(job, new Path(srcpath));      RCFileOutputFormat.setOutputPath(job, new Path(respath));      job.setInputFormatClass(TextInputFormat.class);      job.setOutputFormatClass(RCFileOutputFormat.class);      job.setMapOutputKeyClass(LongWritable.class);      job.setMapOutputValueClass(BytesRefArrayWritable.class);      job.setMapperClass(OutPutTestMapper.class);      conf.set("date", line.getOptionValue(DATE));  //设置压缩参数      conf.setBoolean("mapred.output.compress", true);      conf.set("mapred.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");      code = (job.waitForCompletion(true)) ? 0 : 1;  public class OutPutTestMapper              extends Mapper<LongWritable, Text, LongWritable, BytesRefArrayWritable> {  @Override  public void map(LongWritable key, Text value, Context context)        throws IOException, InterruptedException {      String line = value.toString();      String day = context.getConfiguration().get("date");      if (!line.equals("")) {          String[] lines = line.split(" ", -1);          if (lines.length > 3) {              String time_temp = lines[1];              String times = timeStampDate(time_temp);              String d = times.substring(0, 10);              if (day.equals(d)) {                  byte[][] record = {lines[0].getBytes("UTF-8"),                                    lines[1].getBytes("UTF-8"),lines[2].getBytes("UTF-8"),                                    lines[3].getBytes("UTF-8")};                  }                BytesRefArrayWritable bytes = new BytesRefArrayWritable(record.length);                  for (int i = 0; i < record.length; i++) {                      BytesRefWritable cu = new BytesRefWritable(record[i], 0, record[i].length);                      bytes.set(i, cu);                  }                  context.write(key, bytes);              }          }      }  }