MapReduce编程实现txt文件中的内容导入HBase

来源：互联网发布：淘宝盗图技巧编辑：程序博客网时间：2024/06/05 19:43
一、创建java项目。

写入代码，如下：
[java] view plain copy
 print?
package translate1;  
import java.io.IOException;  
import org.apache.hadoop.conf.*;  
import org.apache.hadoop.fs.Path;  
import  org.apache.hadoop.mapreduce.*;  
import org.apache.hadoop.io.LongWritable;  
import  org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.lib.input.*;  
import org.apache.hadoop.hbase.HBaseConfiguration;  
import org.apache.hadoop.hbase.io.*;  
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;  
import org.apache.hadoop.hbase.client.Put;  
import org.apache.hadoop.hbase.util.Bytes;  
public class translate1 {      
       public static Job createSubmittableJob(Configuration conf, String[] args)throws IOException {  
                       String tableName = args[0];  
                       Path inputDir = new Path(args[1]);  
                       @SuppressWarnings("deprecation")  
                       Job job = new Job (conf, "hac_chapter2_recipe3");  
                       job.setJarByClass(HourlyImporter.class);  
                       FileInputFormat.setInputPaths(job, inputDir);   
                      job.setMapperClass(HourlyImporter.class);  
                      TableMapReduceUtil.initTableReducerJob(tableName, null, job);   
                      job.setNumReduceTasks(0);  
                      TableMapReduceUtil.addDependencyJars(job);  
                      return job;   
                              }   
       public static void main(String[] args)throws Exception {  
                      Configuration conf = HBaseConfiguration.create();  
                      Job job = createSubmittableJob(conf, args);  
                      System.exit (job.waitForCompletion(true) ? 0 : 1);  
                              }  
                     }  
      class HourlyImporter extends Mapper<LongWritable, Text, ImmutableBytesWritable, Put> {  
         private long ts;  
         static byte[] family = Bytes.toBytes("n");  
         @Override  
         protected void setup(Context context) {   
         ts = System.currentTimeMillis();   
          }  
                     public static String change(String str,int n,boolean j){  
                               if(str==null||str.length()>=n) return str;  
                                String s="";  
                               for(int i=str.length();i<n;i++)  
                                      s+="0";  
                              if(j) return s+str;  
                              else    return str+s;    
                              }  
                            @SuppressWarnings("deprecation")  
                    public void map(LongWritable offset, Text value, Context context)throws IOException {  
                           try {  
                                     String line = value.toString();  
                                     String stationID = line.substring(0, 4);  
                                     String month = line.substring(5, 7);  
                                     String day = line.substring(7, 9);  
                                     String rowkey = stationID + month + day;  
                                     byte[] bRowKey = Bytes.toBytes(rowkey);  
                                     ImmutableBytesWritable rowKey =  new ImmutableBytesWritable(bRowKey);  
                                     Put p = new Put(bRowKey);  
                                     for (int i = 1; i < 4 ; i++) {  
                                           String columnI ="v" + change(String.valueOf(i),2,true);   
                                           int beginIndex = i * 2 + 8;  
                                           String valueI =line.substring(beginIndex, beginIndex + 2).trim();  
                                           p.add(family, Bytes.toBytes(columnI),ts, Bytes.toBytes(valueI));  
                                            }  
                                    context.write(rowKey, p);  
                               }catch (InterruptedException e) {  
                                    e.printStackTrace();   
                               }   
                          }   
              }   
二、上传txt文件到HDFS系统

HDFS中的文件内容为：
三、在HBase终端上创建HBase表格

创建时只需指定要创建表格的表名和列族名
四、配置java项目执行的参数

配置内容如图：
五、执行结果

如图：
阅读全文
0 0