HBASE(五 MapReduce)

Hbase 也可以做一些MapReduce操作

Hbase的MaprReuce 无非三种 ：

1. HDFS 中的数据 成为 Hbase 的某个表的某一列
2. HBase中的某一列 成为HDFS 中的数据
3. HBase某一表某列 加工 流入 HBase另一表中某列

实现Demo如下 ：

1.创建两个表 插入模板数据

public class HbaseMR {    private static Configuration conf;    private static Connection conn;    private static Admin admin;    static{        conf = HBaseConfiguration.create();        conf.set("hbase.zookeeper.quorum","hadoop01:2181,hadoop02:2181,hadoop03:2181");        try {          conn = ConnectionFactory.createConnection(conf);        } catch (IOException e) {            e.printStackTrace();        }    }    public static void initTable(){        try {            //创建两个表            admin = conn.getAdmin();            HTableDescriptor word = new HTableDescriptor(TableName.valueOf("word"));            HTableDescriptor stat = new HTableDescriptor(TableName.valueOf("stat"));            HColumnDescriptor content = new HColumnDescriptor("content");            word.addFamily(content);            stat.addFamily(content);            admin.createTable(word);            admin.createTable(stat);            //初始化第一个表的数据            Table table = conn.getTable(TableName.valueOf("word"));            table.setAutoFlushTo(false);            table.setWriteBufferSize(5);            List<Put> lp = new ArrayList<Put>();            Put p1 = new Put(Bytes.toBytes("1"));            p1.add("content".getBytes(), "info".getBytes(), ("The Apache Hadoop software library is a framework").getBytes());            lp.add(p1);            Put p2 = new Put(Bytes.toBytes("2"));            p2.add("content".getBytes(),"info".getBytes(),("The common utilities that support the other Hadoop modules").getBytes());            lp.add(p2);            Put p3 = new Put(Bytes.toBytes("3"));            p3.add("content".getBytes(), "info".getBytes(),("Hadoop by reading the documentation").getBytes());            lp.add(p3);            Put p4 = new Put(Bytes.toBytes("4"));            p4.add("content".getBytes(), "info".getBytes(),("Hadoop from the release page").getBytes());            lp.add(p4);            Put p5 = new Put(Bytes.toBytes("5"));            p5.add("content".getBytes(), "info".getBytes(),("Hadoop on the mailing list").getBytes());            lp.add(p5);            table.put(lp);            table.flushCommits();        } catch (Exception e) {            e.printStackTrace();        }    }    }

**2.Mapper 要继承TableMapper

public class HbaseMapper extends TableMapper<Text,IntWritable> {    @Override    protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {    //通过 value获取某一列族中的某一列 进行加工         byte[] l = value.getValue(Bytes.toBytes("content"), Bytes.toBytes("info"));        String line = new String(l);        String[] split = line.split(" ");        for (String s : split) {            context.write(new Text(s),new IntWritable(1));        }    }}

**3.Reduce 类 要继承TableReduce

public class HbaseReduce extends TableReducer<Text,IntWritable,ImmutableBytesWritable>{    @Override    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {        int sum = 0;        for (IntWritable value : values) {            int i  = Integer.parseInt(value.toString());            sum =sum+i;        }        Put put = new Put(Bytes.toBytes(key.toString()));        put.add(Bytes.toBytes("content"),Bytes.toBytes("info"),Bytes.toBytes(String.valueOf(sum)));        context.write(new ImmutableBytesWritable(Bytes.toBytes(key.toString())),put);    }}

4.重点:Driver类

    public static void main(String[] args) throws Exception {        Configuration conf = HBaseConfiguration.create();        conf.set("hbase.zookeeper.quorum","hadoop01:2181,hadoop02:2181,hadoop03:2181");        Job job = Job.getInstance(conf);        job.setJarByClass(HbaseDriver.class);        //初始化mapper任务 相当于设置mapper类         //参数 分别是: Hbase来源表名，new Scan（）,Mapper类，输出key，输出value，job        TableMapReduceUtil.initTableMapperJob("word",new Scan(),HbaseMapper.class,Text.class,IntWritable.class,job);        //初始化reduce任务 相当于设置reduce类         //参数 分别是: Hbase目的表名，Reduce类，job         jobTableMapReduceUtil.initTableReducerJob("stat",HbaseReduce.class,job);       job.waitForCompletion(true);    }