MapReduce从HBase多路径导出数据到Hive

package com.zz.hbase.ccrc;import java.io.IOException;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Map.Entry;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.hbase.HBaseConfiguration;import org.apache.hadoop.hbase.client.Result;import org.apache.hadoop.hbase.client.Scan;import org.apache.hadoop.hbase.io.ImmutableBytesWritable;import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;import org.apache.hadoop.hbase.mapreduce.TableMapper;import org.apache.hadoop.hbase.util.Bytes;import org.apache.hadoop.io.NullWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import com.zz.hive.Constants;import com.zz.util.DateUtil;public class HBaseToHiveJob {    public static class HBaseToHiveMapper extends TableMapper<Text, Text> {        @Override        protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {            StringBuilder sb = new StringBuilder("");            Map<String, String> m = new HashMap<String, String>();            List<String> temp = new ArrayList<String>();            List<String> a = new ArrayList<String>();            String sp = "-";            String tableName = "";            String str = "";            String keyStr = "";            for (Entry<byte[], byte[]> entry : value.getFamilyMap("aaaa".getBytes()).entrySet()) {                str = new String(entry.getValue());                if (str != null) {                    keyStr = new String(entry.getKey());                    if ("event".equals(keyStr)) {                        tableName = str;                    } else {                        m.put(keyStr, str);                    }                }            }            sb = sort(tableName, m, sb, temp, sp, a);            context.write(new Text(tableName), new Text(new String(sb)));        }        /**         * 按照event值类型重新组合数据存储到hdfs上         * @param tableName         * @param map         * @param sb         * @param t         * @param sp         * @param a         * @return         */        private StringBuilder sort(String tableName, Map<String, String> map, StringBuilder sb, List<String> t, String sp, List<String> a) {            if("Summary".equals(tableName)) {                a = com.zz.option.Constants.SUMMARY_COLUMN;//其中一个表定义的好的List，下面也是一样的                return handler(a, t, sb, sp, map);            }            return sb;        }        /**         * 每条数据的各个值之间加上分隔符，值为null的用“-”占位         * @param a         * @param t         * @param sb         * @param sp         * @param m         * @return         */        private StringBuilder handler(List<String> a, List<String> t, StringBuilder sb, String sp, Map<String, String> m) {            String mapValue;            for (String val : a) {                mapValue = m.get(val);                sb.append(mapValue != null ? mapValue : sp);                sb.append(Constants.BLANK_CHAR);            }            return sb.deleteCharAt(sb.length() - 1);        }    }    public static class HBaseToHiveReducer extends Reducer<Text, Text, NullWritable, Text> {        private MultipleOutputs<NullWritable, Text> out;        @Override        protected void setup(Context context) {            out = new MultipleOutputs<NullWritable, Text>(context);        }        @Override        protected void reduce(Text key, Iterable<Text> values, Context context)throws IOException, InterruptedException {            String tableName = key.toString();            String path = tableName + "/" + tableName;            for (Text val : values) {                out.write(NullWritable.get(), new Text(val), path);            }        }        @Override        protected void cleanup(Context context) throws IOException, InterruptedException {            out.close();        }    }    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {        Configuration conf = HBaseConfiguration.create();        Job job = Job.getInstance(conf, "Fetch HBase2Hive" + DateUtil.getNowTime());        job.setJarByClass(HBaseToHiveJob.class);        Scan scan = new Scan();        scan.setCaching(500);        scan.setCacheBlocks(false);        scan.addFamily(Bytes.toBytes("aaaa"));        Path output = new Path(args[0]);        final FileSystem fileSystem = output.getFileSystem(conf);        fileSystem.delete(output, true);        TableMapReduceUtil.initTableMapperJob("Fetch", scan, HBaseToHiveMapper.class, Text.class, Text.class, job, false);        job.setReducerClass(HBaseToHiveReducer.class);        job.setOutputFormatClass(TextOutputFormat.class);        TextOutputFormat.setOutputPath(job, output);        job.setMapOutputKeyClass(Text.class);        job.setMapOutputValueClass(Text.class);        job.setOutputKeyClass(NullWritable.class);        job.setOutputValueClass(Text.class);        System.exit(job.waitForCompletion(true) ? 0 : 1);    }}