hbase bulkload

bulkload的方式导入数据是hbase的一项相当好的数据导入工具，特别适合做为新系统的历史数据导入工具！hbase本身也封装了相关的类importtsv，官网有简单的介绍http://hbase.apache.org/bulk-loads.html。

      这里我要说明的是如何去快速定制一些适合自己应用的bulkload。

      我们一般需要运行的数据有几种格式，txt的用的最普遍，采用lzo压缩过的txt更专业一些，这里举例lzo格式的源文件。以下代码生成hfile

[java] view plaincopyprint?

1. package com.sina.hbase.mr; 
2.  
3. import java.io.IOException; 
4. import java.util.Date; 
5. import org.apache.hadoop.conf.Configuration; 
6. import org.apache.hadoop.fs.Path; 
7. import org.apache.hadoop.hbase.HBaseConfiguration; 
8. import org.apache.hadoop.hbase.KeyValue; 
9. import org.apache.hadoop.hbase.client.HTable; 
10. import org.apache.hadoop.hbase.io.ImmutableBytesWritable; 
11. import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; 
12. import org.apache.hadoop.hbase.util.Bytes; 
13. import org.apache.hadoop.io.LongWritable; 
14. import org.apache.hadoop.io.Text; 
15. import org.apache.hadoop.mapreduce.Job; 
16. import org.apache.hadoop.mapreduce.Mapper; 
17. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
18. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
19. import com.hadoop.mapreduce.LzoTextInputFormat; 
20. import com.sina.hbase.connection.ConnectionPool; 
21. import com.sina.hbase.utils.DataOptUtil; 
22. import com.sina.hbase.utils.Util; 
23.  
24.  
25. public class BulkLoad { 
26.  
27.     public staticclass ***Mapper extends 
28.             Mapper<LongWritable, Text, ImmutableBytesWritable, KeyValue> { 
29.  
30.         @Override 
31.         protected void map(LongWritable key, Text value, Context context) 
32.                 throws IOException, InterruptedException { 
33.             // 检查并初始化数据对象 
34.             *** p = Util.checkAndBuild(value.toString()); 
35.  
36.             if (p != null) { 
37.                 byte[] row = Bytes.toBytes(p.getUid()); 
38.                 ImmutableBytesWritable k = new ImmutableBytesWritable(row); 
39.                 KeyValue kv = new KeyValue(row,"c".getBytes(), "c".getBytes(), 
40.                         p.toByteArray()); 
41.                 context.write(k, kv); 
42.  
43.             } 
44.         } 
45.     } 
46.  
47.      
48.  
49.     /**
50.      * 通过表名决定使用哪种Mapper，如果表名不存在则返回null
51.      *
52.      * @param tableName
53.      * @return
54.      */ 
55.  
56.     @SuppressWarnings("rawtypes") 
57.     public static Class<?extends Mapper> decideMapper(String tableName) { 
58.         if (tableName.equals("***")) 
59.             return ***Mapper.class; 
60.          
61.  
62.         return null; 
63.     } 
64.  
65.     public staticvoid main(String[] args) throws Exception { 
66.  
67.         if (args.length !=3) { 
68.             System.err 
69.                     .println("Usage: BulkLoad <inputPath> <hfilePath> <tablename>"); 
70.             System.exit(2); 
71.         } 
72.         Configuration conf = HBaseConfiguration.create(); 
73.         ConnectionPool.init(conf, 1000); 
74.         HTable table = null; 
75.          
76.             table = ConnectionPool.getTable(args[2]); 
77.  
78.         Job job = new Job(conf, "BulkLoad-" + args[2] +"-" 
79.                 + DataOptUtil.Date2LongString(new Date())); 
80.  
81.         // 根据表的不同选择mapper 
82.  
83.         job.setMapperClass(decideMapper(args[2])); 
84.  
85.         job.setJarByClass(BulkLoad.class); 
86.         job.setInputFormatClass(LzoTextInputFormat.class); 
87.  
88.         HFileOutputFormat.configureIncrementalLoad(job, table); 
89.  
90.         FileInputFormat.addInputPath(job, new Path(args[0])); 
91.         FileOutputFormat.setOutputPath(job, 
92.                 Util.RemoveHDFSPath(new Path(args[1]), conf)); 
93.  
94.         System.exit(job.waitForCompletion(true) ?0 : 1); 
95.     } 
96.  
97. } 

package com.sina.hbase.mr;import java.io.IOException;import java.util.Date;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.hbase.HBaseConfiguration;import org.apache.hadoop.hbase.KeyValue;import org.apache.hadoop.hbase.client.HTable;import org.apache.hadoop.hbase.io.ImmutableBytesWritable;import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;import org.apache.hadoop.hbase.util.Bytes;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import com.hadoop.mapreduce.LzoTextInputFormat;import com.sina.hbase.connection.ConnectionPool;import com.sina.hbase.utils.DataOptUtil;import com.sina.hbase.utils.Util;public class BulkLoad {public static class ***Mapper extendsMapper<LongWritable, Text, ImmutableBytesWritable, KeyValue> {@Overrideprotected void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException {// 检查并初始化数据对象*** p = Util.checkAndBuild(value.toString());if (p != null) {byte[] row = Bytes.toBytes(p.getUid());ImmutableBytesWritable k = new ImmutableBytesWritable(row);KeyValue kv = new KeyValue(row, "c".getBytes(), "c".getBytes(),p.toByteArray());context.write(k, kv);}}}/** * 通过表名决定使用哪种Mapper，如果表名不存在则返回null *  * @param tableName * @return */@SuppressWarnings("rawtypes")public static Class<? extends Mapper> decideMapper(String tableName) {if (tableName.equals("***"))return ***Mapper.class;return null;}public static void main(String[] args) throws Exception {if (args.length != 3) {System.err.println("Usage: BulkLoad <inputPath> <hfilePath> <tablename>");System.exit(2);}Configuration conf = HBaseConfiguration.create();ConnectionPool.init(conf, 1000);HTable table = null;table = ConnectionPool.getTable(args[2]);Job job = new Job(conf, "BulkLoad-" + args[2] + "-"+ DataOptUtil.Date2LongString(new Date()));// 根据表的不同选择mapperjob.setMapperClass(decideMapper(args[2]));job.setJarByClass(BulkLoad.class);job.setInputFormatClass(LzoTextInputFormat.class);HFileOutputFormat.configureIncrementalLoad(job, table);FileInputFormat.addInputPath(job, new Path(args[0]));FileOutputFormat.setOutputPath(job,Util.RemoveHDFSPath(new Path(args[1]), conf));System.exit(job.waitForCompletion(true) ? 0 : 1);}}

      以上的源代码很简单，但是够用。需要做一些说明的是：

     1、一定记得在建表时做region的预切分，HFileOutputFormat.configureIncrementalLoad方法会根据region的数量来觉得reduce的数量以及每个reduce覆盖的rowkey范围。否则当个reduce过大，任务处理不均衡。

     2、单个rowkey下的子列不要过多，否则在reduce阶段排序的时候会造成oom，有一种办法是通过二次排序来避免reduce阶段的排序，看应用而定。

     3、该代码执行完后需要将hdfs中生成好的hfile写入到hbase表中。采用hadoop jar hbase-version.jar completebulkload /hfilepath tablename 命令实现。

     4、导入hadoop-lzo的jar包，才有LzoTextInputFormat这个类。