spring-hadoop之操作hbase

本文转载自：spring-hadoop之操作hbase

Srping对于属于java web技术的程序员都不会陌生，jdbcTemplate更是用的熟之又熟，下面我们来认识一下Spring大家庭的新成员：Spring-data-hadoop项目。Spring-hadoop这个项目应该是在Spring Data项目的一部分（Srping data其余还包括把Spring和JDBC，REST，主流的NoSQL结合起来了）。其实再一想，Spring和Hadoop结合会发生什么呢，其实就是把Hadoop组件的配置，任务部署之类的东西都统一到Spring的bean管理里去了。

具体操作如下：

1、下载并导入需要的jar包：

<classpathentry kind="lib" path="WebRoot/WEB-INF/lib/commons-configuration-1.6.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/commons-lang-2.5.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/commons-logging-1.1.1.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/dom4j-1.6.1.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/guava-11.0.2.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/hadoop-auth-2.0.0-cdh4.4.0.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/hadoop-common-2.0.0-cdh4.4.0.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/hadoop-mapreduce-client-core-2.0.0-cdh4.4.0.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/hbase-0.94.6-cdh4.4.0-security.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/log4j-1.2.17.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/protobuf-java-2.4.0a.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/slf4j-api-1.6.1.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/slf4j-log4j12-1.6.1.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-aop-4.0.3.RELEASE.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-aspects-4.0.3.RELEASE.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-beans-4.0.3.RELEASE.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-context-4.0.3.RELEASE.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-context-support-4.0.3.RELEASE.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-core-4.0.3.RELEASE.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-data-hadoop-2.0.0.RC2-cdh4.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-data-hadoop-batch-2.0.0.RC2-cdh4.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-data-hadoop-core-2.0.0.RC2-cdh4.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-data-hadoop-store-2.0.0.RC2-cdh4.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-expression-4.0.3.RELEASE.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-tx-4.0.3.RELEASE.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/spring-web-4.0.3.RELEASE.jar"/>    <classpathentry kind="lib" path="WebRoot/WEB-INF/lib/zookeeper-3.4.5-cdh4.4.0.jar"/>

2、配置web.xml：

 1 <?xml version="1.0" encoding="UTF-8"?> 2 <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/j2ee" 3     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 4     xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee http://java.sun.com/xml/ns/j2ee/web-app_2_5.xsd"> 5     <context-param> 6         <param-name>contextConfigLocation</param-name> 7         <param-value>classpath:beans.xml</param-value> 8     </context-param> 9     10     11     <listener>12         <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>13     </listener>14     <!-- Spring 刷新Introspector防止内存泄露 -->15     <listener>16         <listener-class>org.springframework.web.util.IntrospectorCleanupListener</listener-class>17     </listener>18     19     <welcome-file-list>20         <welcome-file>login.jsp</welcome-file>21     </welcome-file-list>22 </web-app>

3、配置beans.xml:

 1 <?xml version="1.0" encoding="UTF-8"?> 2 <beans xmlns="http://www.springframework.org/schema/beans" 3        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 4         xmlns:context="http://www.springframework.org/schema/context" 5         xmlns:p="http://www.springframework.org/schema/p" 6         xmlns:aop="http://www.springframework.org/schema/aop" 7         xmlns:hdp="http://www.springframework.org/schema/hadoop" 8         xsi:schemaLocation=" 9             http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd10             http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-4.0.xsd11             http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-4.0.xsd12             http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop-2.0.xsd13             ">14     15     <!-- 自动扫描beans+注解功能注册 -->16     <context:component-scan base-package="net.xinhong"/>17     18     <hdp:configuration resources="classpath:/hbase-site.xml" />19     20     <hdp:hbase-configuration configuration-ref="hadoopConfiguration" />21     22     <bean id="hbaseTemplate" class="org.springframework.data.hadoop.hbase.HbaseTemplate" p:configuration-ref="hbaseConfiguration"/>23     24 </beans>

4、将hbase-site.xml配置文件拷贝到src目录下，参考内容如下：

 1 <?xml version="1.0" encoding="UTF-8"?> 2 <configuration> 3   <property> 4     <name>hbase.rootdir</name> 5     <value>hdfs://nameservice1/hbase</value> 6   </property> 7   <property> 8     <name>hbase.client.write.buffer</name> 9     <value>62914560</value>10   </property>11   <property>12     <name>hbase.client.pause</name>13     <value>1000</value>14   </property>15   <property>16     <name>hbase.client.retries.number</name>17     <value>10</value>18   </property>19   <property>20     <name>hbase.client.scanner.caching</name>21     <value>1</value>22   </property>23   <property>24     <name>hbase.client.keyvalue.maxsize</name>25     <value>62914560</value>26   </property>27   <property>28     <name>hbase.rpc.timeout</name>29     <value>60000</value>30   </property>31   <property>32     <name>hbase.security.authentication</name>33     <value>simple</value>34   </property>35   <property>36     <name>zookeeper.session.timeout</name>37     <value>60000</value>38   </property>39   <property>40     <name>zookeeper.znode.parent</name>41     <value>/hbase</value>42   </property>43   <property>44     <name>zookeeper.znode.rootserver</name>45     <value>root-region-server</value>46   </property>47   <property>48     <name>hbase.zookeeper.quorum</name>49     <value>xinhong-hadoop-56,xinhong-hadoop-52,xinhong-hadoop-53</value>50   </property>51   <property>52     <name>hbase.zookeeper.property.clientPort</name>53     <value>2181</value>54   </property>55 </configuration>

5、hbase模板的demo:

查询列族下的列的数据：

1 public List<String> find(String tableName,String family,String cloumn){2         List<String> rows = hbaseTemplate.find(tableName, family,cloumn, new RowMapper<String>() {3             public String mapRow(Result result, int rowNum) throws Exception {4                 return Bytes.toString(result.getRow());5             }6         });7         return rows;8     }

查询指定行健的一列数据：

1 public String get(String tableName,String family,String cloumn,String rowKey){2         String context = hbaseTemplate.get(tableName, "NCEP_p_wa_2014032212_tp_006.nc", family, cloumn, new RowMapper<String>() {3             public String mapRow(Result result, int rowNum) throws Exception {4                 return Bytes.toString(result.value());5               }6             });7         return context;8     }