mapreduce读取文件内容输出到hbase

来源:互联网 发布:删了windows.old就崩溃 编辑:程序博客网 时间:2024/05/16 10:24


package hadoop;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.hbase.HBaseConfiguration;import org.apache.hadoop.hbase.client.Put;import org.apache.hadoop.hbase.io.ImmutableBytesWritable;import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;import org.apache.hadoop.hbase.mapreduce.TableReducer;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;public class FileToHbase {static class FileMapper extends Mapper<LongWritable, Text, Text, IntWritable>{@Overrideprotected void map(LongWritable key, Text value,Context context)throws IOException, InterruptedException {String line = value.toString();String[] lines = line.split(" ");for(String string:lines){context.write(new Text(string), new IntWritable(1));}}}static class FileReduce extends TableReducer<Text, IntWritable, ImmutableBytesWritable>{@Overrideprotected void reduce(Text key, Iterable<IntWritable> values,Context context)throws IOException, InterruptedException {int sum=0;for(IntWritable ints:values){sum += ints.get();}Put put = new Put(("row"+key).getBytes());put.add("f".getBytes(), key.getBytes(), (sum+"").getBytes());context.write(new ImmutableBytesWritable("result".getBytes()), put);}}public static void main(String[] args) throws Exception {Configuration conf = HBaseConfiguration.create();conf.set("hbase.zookeeper.quorum", "192.168.10.250");conf.set("hbase.zookeeper.property.clientPort", "2181");Job job = Job.getInstance(conf,"hbasedemo");job.setJarByClass(FileToHbase.class);FileInputFormat.addInputPath(job, new Path("/test/test.txt"));job.setMapperClass(FileMapper.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);TableMapReduceUtil.initTableReducerJob("result", FileReduce.class, job);boolean b = job.waitForCompletion(true);if(!b){System.exit(1);}}}


阅读全文
0 0
原创粉丝点击