hadoop hbase

本来要离开厦门回福州的，结果被人介绍到银行大厦聊了一下，他说：要不来试几个月吧，几个月后你确定要回福州做事就回去呗。

我觉得人挺好的，于是就进了这公司。挺有意思的。

个人笔记，IBM合作的一个项目做了一个简单功能部分记录下。 

package com.ibm.bi.ccb.TC_OLTP_06;import java.io.IOException;import java.util.ArrayList;import java.util.List;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.hbase.HBaseConfiguration;import org.apache.hadoop.hbase.HColumnDescriptor;import org.apache.hadoop.hbase.HTableDescriptor;import org.apache.hadoop.hbase.KeyValue;import org.apache.hadoop.hbase.MasterNotRunningException;import org.apache.hadoop.hbase.ZooKeeperConnectionException;import org.apache.hadoop.hbase.client.Delete;import org.apache.hadoop.hbase.client.Get;import org.apache.hadoop.hbase.client.HBaseAdmin;import org.apache.hadoop.hbase.client.HTable;import org.apache.hadoop.hbase.client.Put;import org.apache.hadoop.hbase.client.Result;import org.apache.hadoop.hbase.client.ResultScanner;import org.apache.hadoop.hbase.client.Scan;import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;import org.apache.hadoop.hbase.util.Bytes;/** * 操作HBASE工具类 *  * @author ZengWenfeng * @date 2013-03-24 */public class HBaseUtil{/** * 配置文件 *  * @author ZengWenfeng * @date 2013-03-24 */private static Configuration conf = null;/** * 加载配置文件 *  * @author ZengWenfeng * @date 2013-03-24 */static {conf = HBaseConfiguration.create();//<configuration>//  <property>//      <name>hbase.cluster.distributed</name>//      <value>true</value>//   </property>////   <property>//     <name>hbase.rootdir</name>//     <value>hdfs://masternode:8020/hbase</value>//   </property>////   <property>//     <name>hbase.zookeeper.quorum</name>//     <value>masternode</value>// </property>//</configuration>conf.set("hbase.cluster.distributed", "true");conf.set("hbase.rootdir", "hdfs://masternode:8020/hbase");conf.set("hbase.zookeeper.quorum", "slavenode6");//conf.set("hbase.master", "gwnode1:8020");//conf.set("hbase.zookeeper.quorum", "172.16.123.210");  //conf.set("hbase.zookeeper.property.clientPort", "8020"); }public void loadFiles(){try{Path hfofDir = null;HTable table = null;LoadIncrementalHFiles loadFiles = new LoadIncrementalHFiles(conf);loadFiles.doBulkLoad(hfofDir, table);}catch (Exception e){e.printStackTrace();}}/** * 创建表 *  * @author ZengWenfeng * @date 2013-03-24 * @param tabelNam 表名 * @param cfs 列族 * @throws IOException */public void createTable(String tabelName, List<String> cfs) throws IOException {HBaseAdmin admin = new HBaseAdmin(conf);if (admin.tableExists(tabelName)) {System.out.println("table exist.");} else {HTableDescriptor tableDesc = new HTableDescriptor(tabelName);if (cfs.isEmpty()) {admin.close();return;}for (int i = 0; i < cfs.size(); i++) {tableDesc.addFamily(new HColumnDescriptor(cfs.get(i)));}admin.createTable(tableDesc);}admin.close();}/** * 删除表 *  * @author ZengWenfeng * @date 2013-03-24 * @param tableName * @throws IOException */public void deleteTable(String tableName) throws IOException {HBaseAdmin admin = new HBaseAdmin(conf);try {admin.disableTable(tableName);admin.deleteTable(tableName);System.out.println("删除成功");} catch (MasterNotRunningException e) {e.printStackTrace();} catch (ZooKeeperConnectionException e) {e.printStackTrace();} finally {admin.close();}}/** * 插入记录 *  * @author ZengWenfeng * @date 2013-03-24 * @param tableName * @param cfs * @throws IOException */public void writeRow(String tableName, List<String> cfs) throws IOException {HTable table = new HTable(conf, tableName);try {Put put = new Put(Bytes.toBytes(cfs.get(0)));put.add(Bytes.toBytes(cfs.get(1)), Bytes.toBytes(cfs.get(2)), Bytes.toBytes(cfs.get(3)));table.put(put);//table.put(puts);System.out.println("插入成功");}catch (IOException e) {e.printStackTrace();} finally {table.close();}}/** * 更新记录 *  * @author ZengWenfeng * @date 2013-03-24 * @param tableName * @param cfs * @throws IOException */public void updateRow(String tableName, List<String> cfs) throws IOException {HTable table = new HTable(conf, tableName);try {Put put = new Put(Bytes.toBytes(cfs.get(0)));put.add(Bytes.toBytes(cfs.get(1)), Bytes.toBytes(cfs.get(2)), Bytes.toBytes(cfs.get(3)));table.put(put);System.out.println("success");} catch (IOException e) {e.printStackTrace();}finally {table.close();}}/** * 删除记录 *  * @author ZengWenfeng * @date 2013-03-24 * @param tableName * @param rowkey * @throws IOException */public void deleteRow(String tableName, String rowkey) throws IOException {HTable table = new HTable(conf, tableName);List<Delete> list = new ArrayList<Delete>();Delete d1 = new Delete(rowkey.getBytes());list.add(d1);table.delete(list);System.out.println("删除成功");table.close();}public void query(String tableName, String[] rowKeys) throws IOException{if (tableName == null || tableName.equals(tableName)){return;}}/** * 查询 *  * @param tableName * @param rowkey */public void selectRow(String tableName, String rowKey) throws IOException {HTable table = new HTable(conf, tableName);Get g = new Get(rowKey.getBytes());Result rs = table.get(g);if (rs != null && rs.size() > 0){for (KeyValue kv : rs.raw()){System.out.println(new String(kv.getRow()) + "|"+ new String(kv.getFamily()) + ":"+ new String(kv.getQualifier()) + "|"+ new String(kv.getValue()) + "|" + kv.getTimestamp());}}table.close();}/** * 扫描表 *  * @param tableName * @throws IOException */public void scan(String tableName) throws IOException {HTable table = new HTable(conf, tableName);try {Scan s = new Scan();ResultScanner rs = table.getScanner(s);for (Result r : rs) {KeyValue[] kv = r.raw();for (int i = 0; i < kv.length; i++) {System.out.println(new String(kv[i].getRow()) + "|"+ new String(kv[i].getFamily()) + ":"+ new String(kv[i].getQualifier()) + "|"+ new String(kv[i].getValue()) + "|"+ kv[i].getTimestamp());}}}catch (IOException e) {e.printStackTrace();}finally {table.close();}}/** * 仅供测试 *  * @author ZengWenfeng * @date 2013-03-24 * @param args * @throws IOException */public static void main(String[] args) throws IOException {HBaseUtil hu = new HBaseUtil();hu.deleteTable("zwf01");//ArrayList<String> al = new ArrayList<String>();//al.add("fdsafsd");//al.add("bbb");//hu.createTable("sfds", al);//StringBuffer sb = new StringBuffer();////for (String s : args) //{//sb.append(s + " ");//}////System.out.println(sb.toString());////HBaseUtil hb = new HBaseUtil();////if (args.length <= 1 || args[0].equalsIgnoreCase("help")) //{//System.out.println("--help--");//} //else if (args[0].equalsIgnoreCase("-c") || args[0].equalsIgnoreCase("create")) //{//List<String> strs = new ArrayList<String>();////for (int i = 2; i < args.length; i++)//{//strs.add(args[i]);//}////hb.createTable(args[1], strs);//} //else if (args[0].equalsIgnoreCase("-dt") || args[0].equalsIgnoreCase("delete-table")) //{//hb.deleteTable(args[1]);//}//else if (args[0].equalsIgnoreCase("-i") || args[0].equalsIgnoreCase("insert")) //{//List<String> strs = new ArrayList<String>();//for (int i = 2; i < args.length; i++) //{//strs.add(args[i]);//}////hb.writeRow(args[1], strs);//} //else if (args[0].equalsIgnoreCase("-dr") || args[0].equalsIgnoreCase("delete-row")) //{//hb.deleteRow(args[1], args[2]);//}//else if (args[0].equalsIgnoreCase("-qr") || args[0].equalsIgnoreCase("query-row")) //{//hb.selectRow(args[1], args[2]);//}//else if (args[0].equalsIgnoreCase("-qa") || args[0].equalsIgnoreCase("query-all")) //{//hb.scan(args[1]);//}//else if (args[0].equalsIgnoreCase("-u") || args[0].equalsIgnoreCase("update")) //{//List<String> strs = new ArrayList<String>();////for (int i = 2; i < args.length; i++) //{//strs.add(args[i]);//}////hb.updateRow(args[1], strs);//}}}

package com.ibm.bi.ccb.TC_OLTP_06;import java.io.IOException;/** * @author ZengWenfeng * @date 2013-04-11 */public class Task extends Thread{/** * 表名 */private String tableName;/** * 主键 */private String[] rowKeys;/** * 构造函数 */public Task(){}/** * 构造函数 */public Task(String tableName, String[] rowKeys){this.tableName = tableName;this.rowKeys = rowKeys;}@Overridepublic void run(){System.out.println(Thread.currentThread().getName());if (rowKeys != null && rowKeys.length > 0){for (int i = 0; i < rowKeys.length; i++){HBaseUtil hu = new HBaseUtil();try{hu.selectRow(tableName, rowKeys[i]);}catch (IOException e){e.printStackTrace();}}}else{System.out.println("error");}}}

package com.ibm.bi.ccb.TC_OLTP_06;import java.io.IOException;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.hbase.HBaseConfiguration;import org.apache.hadoop.hbase.KeyValue;import org.apache.hadoop.hbase.client.HTable;import org.apache.hadoop.hbase.client.Result;import org.apache.hadoop.hbase.client.ResultScanner;import org.apache.hadoop.hbase.client.Scan;import org.apache.hadoop.hbase.filter.BinaryComparator;import org.apache.hadoop.hbase.filter.CompareFilter;import org.apache.hadoop.hbase.filter.Filter;import org.apache.hadoop.hbase.filter.FilterList;import org.apache.hadoop.hbase.filter.RowFilter;import org.apache.hadoop.hbase.util.Bytes;import org.slf4j.Logger;import com.ibm.bi.ccb.framework.ResourceManager;/** * TC_OLTP_06 *  * @author ZengWenfeng * @date 2013-04-12 */public class TC_OLTP_06_Task extends Thread{/** * 日志 */private static final Logger logger = ResourceManager.logger(TC_OLTP_06_Task.class);/** * 查询的表名 */private String tableName = "";/** * 多个主键组 */private String rowKeys = "";/** * 开始时间 */private long startTime = 0L;/** * 查询结果集合 */public static List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();/** *  */public TC_OLTP_06_Task(long startTime){this.startTime = startTime;}/** * 根据给定的主键以及表名执行查询 *  * @see dev.toolkit.concurrent.Task#doTask() */public void run(){try{ArrayList<Filter> al = new ArrayList<Filter>();String[] keys = rowKeys.split(",");for (int i = 0; i < keys.length; i++){Filter filter = new RowFilter(CompareFilter.CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(keys[i])));al.add(filter);}FilterList filterList = new FilterList(al);Scan scan = new Scan();scan.setFilter(filterList);Configuration conf = HBaseConfiguration.create();conf.set("hbase.cluster.distributed", "true");conf.set("hbase.rootdir", "hdfs://masternode:8020/hbase");conf.set("hbase.zookeeper.quorum", "slavenode6");HTable table = new HTable(conf, tableName);ResultScanner rs = table.getScanner(scan);list = new ArrayList<Map<String, Object>>();logger.info("Scanning " + tableName + "...");for (Result r : rs) {KeyValue[] kv = r.raw();for (int i = 0; i < kv.length; i++) {Map<String, Object> data = new HashMap<String, Object>();data.put("ROW", new String(kv[i].getRow()));data.put("FAMILY_QUALIFIER", new String(kv[i].getFamily()) + ":" + new String(kv[i].getQualifier()));data.put("VALUE", new String(kv[i].getValue()));data.put("TIME", kv[i].getTimestamp());list.add(data);//System.out.println(new String(kv[i].getRow()) + "|"//+ new String(kv[i].getFamily()) + ":"//+ new String(kv[i].getQualifier()) + "|"//+ new String(kv[i].getValue()) + "|"//+ kv[i].getTimestamp());}}rs.close();long stopTime = System.currentTimeMillis();logger.info("stopTime: " + stopTime + " ms");long executeTime = stopTime - startTime;logger.info("executeTime: " + executeTime + "ms");}catch (IOException e){e.printStackTrace();}}public String getTableName(){return tableName;}public void setTableName(String tableName){this.tableName = tableName;}public String getRowKeys(){return rowKeys;}public void setRowKeys(String rowKeys){this.rowKeys = rowKeys;}public long getStartTime(){return startTime;}public void setStartTime(long startTime){this.startTime = startTime;}}

/** * TC_OLTP_06.java *  * 俗世翩翩少年歌一曲 * 把心声写给青山听 */package com.ibm.bi.ccb.TC_OLTP_06;import java.util.ArrayList;import org.slf4j.Logger;import com.ibm.bi.ccb.framework.ResourceManager;/** * TC_OLTP_06 *  * @author ZengWenfeng * @date 2013-04-12 */public class TC_OLTP_06{/** * 日志 */private static final Logger logger = ResourceManager.logger(TC_OLTP_06.class);/** * 线程数 */private static int threadNumber = 1;/** * 任务数组 */private static TC_OLTP_06_Task[] tasks = null;/** * 构造函数 */public TC_OLTP_06(){logger.info("TC_OLTP_06 initialized");}/** * 主函数,仅供测试 * wot, 1,12,2 *  * @author ZengWenfeng * @date 2013-04-17 * @param args */public static void main(String[] args){//String tableName = "";String rowKeys = "";// one argumentif (args.length == 1){tableName = args[0];}// two argumentselse if (args.length == 2){tableName = args[0];rowKeys = args[1];}// exceptionelse{System.out.println("args is null.");System.out.println("usage: Test <tableName> <rowKey1,rowKey2,...,rowKeyn>");return;}//执行任务excuteTask(tableName, rowKeys);}/** * 执行任务 *  * @author ZengWenfeng * @date 2013-04-17 * @param tableName * @param rowKeys */public static void excuteTask(String tableName, String rowKeys){// 校验表名，为空不能执行查询if (tableName == null || tableName.equals("")){System.out.println("args[0] is table name. it is not null.");System.out.println("usage: Test <tableName> <rowKey1,rowKey2,...,rowKeyn>");return;}// 去重rowKeysrowKeys = removeDuplicateRecords(rowKeys);tasks = new TC_OLTP_06_Task[threadNumber];try{long startTime = System.currentTimeMillis();logger.info("startTime: " + startTime + " ms");for (int i = 0; i < threadNumber; i++){TC_OLTP_06_Task task = new TC_OLTP_06_Task(startTime);task.setTableName(tableName);task.setRowKeys(rowKeys);task.start();}}catch (Exception e){e.printStackTrace();}}/** * 删除重复记录 *  * @author ZengWenfeng * @date 2013-04-11 * @param rowKeys * @return */public static String removeDuplicateRecords(String rowKeys){if (rowKeys.indexOf(",") <= 0){return rowKeys;}String[] keys = rowKeys.split(",");ArrayList<String> al = new ArrayList<String>();//for (int i = 0; i < keys.length; i++){if (!al.contains(keys[i])){al.add(keys[i]);}}//StringBuffer sb = new StringBuffer();for (int j = 0; j < al.size(); j++){sb.append(al.get(j));if (j < al.size() - 1){sb.append(",");}}return sb.toString();}}

package com.ibm.bi.ccb.TC_OLTP_06;import java.io.IOException;import java.util.ArrayList;import org.slf4j.Logger;import com.ibm.bi.ccb.framework.ResourceManager;/** * TC_OLTP_06 *  * @author ZengWenfeng * @date 2013-04-10 */public class Test{/** * 日志 */private static final Logger logger = ResourceManager.logger(Test.class);/** * 最大线程数 *  * @author ZengWenfeng * @date 2013-04-10 */private static int maxThreadNum = 30;/** * 构造函数 *  * @author ZengWenfeng * @date 2013-04-10 */public Test(){logger.info("TC_OLTP_06 initialized");}/** * 删除重复记录 *  * @author ZengWenfeng * @date 2013-04-11 * @param rowKeys * @return */public static String removeDuplicateRecords(String rowKeys){if (rowKeys.indexOf(",") <= 0){return rowKeys;}String[] keys = rowKeys.split(",");ArrayList<String> al = new ArrayList<String>();//for (int i = 0; i < keys.length; i++){if (!al.contains(keys[i])){al.add(keys[i]);}}StringBuffer sb = new StringBuffer();for (int j = 0; j < al.size(); j++){sb.append(al.get(j));if (j < al.size() - 1){sb.append(",");}}return sb.toString();}/** * 仅供测试 *  * @author ZengWenfeng * @date 2013-04-10 * @param args */public static void main(String[] args){//String tableName = "";String rowKeys = "";// one argumentif (args.length == 1){tableName = args[0];}// two argumentselse if (args.length == 2){tableName = args[0];rowKeys = args[1];}// exceptionelse{System.out.println("args is null.");System.out.println("usage: Test <tableName> <rowKey1,rowKey2,...,rowKeyn>");return;}// 校验表名，为空不能执行查询if (tableName == null || tableName.equals("")){System.out.println("args[0] is table name. it is not null.");System.out.println("usage: Test <tableName> <rowKey1,rowKey2,...,rowKeyn>");return;}// 去重rowKeysrowKeys = removeDuplicateRecords(rowKeys);// 当查询的row不为空if (rowKeys != null && rowKeys.length() > 0){//String regex = ",";int index = rowKeys.indexOf(regex);// 查询条件的值为多个时if (index > 0){//String[] keys = rowKeys.split(regex);//int allTasks = keys.length;// 需要分配最大线程数if (allTasks > maxThreadNum){// 83 40 2 3// 求余数3int remainder = allTasks % maxThreadNum;// 每个线程需要至少多少个任务  2int tasksPerThread = allTasks / maxThreadNum;// 根据最大线程数，启动多线程for (int i = 0; i < maxThreadNum; i++){// 00 01 02 ... 39// 40 41 42 ... 79// 80 81 82int threadNum = 0;if (i < remainder){threadNum += tasksPerThread + 1;}else{threadNum += tasksPerThread;}String[] ids = new String[threadNum];for (int j = 0; j < threadNum; j++){ids[j] = keys[i + j * maxThreadNum];}Task t = new Task(tableName, ids);t.start();}}// 需要分配线程小于等于最大线程数else{// 根据任务数，启动多线程for (int i = 0; i < allTasks; i++){String[] ids = new String[1];ids[0] = keys[i];Task t = new Task(tableName, ids);t.start();}}}// 当查询条件值有且只有一个else{HBaseUtil hbUtil = new HBaseUtil();try{hbUtil.selectRow(tableName, rowKeys);}catch (IOException e){e.printStackTrace();}}}// 当查询条件为空，查询所有数据,执行scan 'tableName'else{HBaseUtil hbUtil = new HBaseUtil();try{hbUtil.scan(tableName);}catch (IOException e){e.printStackTrace();}}}}

package com.ibm.bi.ccb.TC_OLTP_06;final class Timer {private static  Long startTime;private static Long endTime;public static Long getStartTime() {return startTime;}public static void setStartTime(Long startTime){Timer.startTime = startTime;}public static Long getEndTime(){return endTime;}public static void setEndTime(Long endTime){Timer.endTime = endTime;}public static long stopTimerAndGetElapseTime() {Long startTime =getStartTime();Long endTime = System.currentTimeMillis();// System.out.println("End time: " + endTime +// " for current thread id: "// + Thread.currentThread().getId());if (startTime == null){return -1;}else{return endTime - startTime;}}}