Spring 配置 hive

来源：互联网发布：无线mac地址修改器编辑：程序博客网时间：2024/06/02 03:32

之前试着搞了一下hbase，现在再来试一下hive的配置使用。

配置

pom.xml

内容没有给全，主要贴出了hbase需要的，其他的根据个人需要而添加

      <properties>            <spring.version>4.1.6.RELEASE</spring.version>            <spring.hadoop.version>2.4.0.RELEASE</spring.hadoop.version>            <hadoop.version>2.6.0</hadoop.version>            <hive.version>1.0.0</hive.version>      </properties>      <dependency>        <groupId>org.springframework</groupId>        <artifactId>spring-tx</artifactId>        <version>${spring.version}</version>      </dependency>    <dependency>      <groupId>org.springframework</groupId>      <artifactId>spring-jdbc</artifactId>      <version>${spring.version}</version>    </dependency>    <dependency>      <groupId>org.springframework.data</groupId>      <artifactId>spring-data-hadoop</artifactId>      <version>${spring.hadoop.version}</version>      <exclusions>        <exclusion>          <groupId>org.springframework</groupId>          <artifactId>spring-context-support</artifactId>        </exclusion>        <exclusion>          <groupId>org.slf4j</groupId>          <artifactId>slf4j-log4j12</artifactId>        </exclusion>      </exclusions>    </dependency>    <!-- hive -->    <dependency>      <groupId>org.apache.hive</groupId>      <artifactId>hive-metastore</artifactId>      <version>${hive.version}</version>    </dependency>    <dependency>      <groupId>org.apache.hive</groupId>      <artifactId>hive-service</artifactId>      <version>${hive.version}</version>    </dependency>    <dependency>      <groupId>org.apache.thrift</groupId>      <artifactId>libfb303</artifactId>      <version>0.9.1</version>    </dependency>    <dependency>      <groupId>org.apache.hive</groupId>      <artifactId>hive-common</artifactId>      <version>${hive.version}</version>      <scope>runtime</scope>    </dependency>    <dependency>      <groupId>org.apache.hive</groupId>      <artifactId>hive-jdbc</artifactId>      <version>1.1.0</version>    </dependency>    <dependency>      <groupId>org.apache.hive</groupId>      <artifactId>hive-shims</artifactId>      <version>${hive.version}</version>    </dependency>    <dependency>      <groupId>org.apache.hive</groupId>      <artifactId>hive-serde</artifactId>      <version>${hive.version}</version>      <scope>runtime</scope>    </dependency>    <dependency>      <groupId>org.apache.hive</groupId>      <artifactId>hive-contrib</artifactId>      <version>${hive.version}</version>    </dependency>

web.xml

<?xml version="1.0" encoding="UTF-8"?><web-app version="3.0"         xmlns="http://java.sun.com/xml/ns/javaee"         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"         xsi:schemaLocation="http://java.sun.com/xml/ns/javaee         http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd">  <display-name>${maven.project.name}</display-name>  <listener>    <listener-class>org.springframework.web.util.IntrospectorCleanupListener</listener-class>  </listener>  <!-- ========================================================= -->  <!-- Spring配置 -->  <!-- ========================================================= -->  <listener>    <description>spring监听器</description>    <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>  </listener>  <context-param>    <param-name>contextConfigLocation</param-name>    <param-value>classpath*:/spring/*.xml</param-value>  </context-param>  <!-- ========================================== -->  <!-- 字符集过滤器，对request和response中的字符编码 -->  <!-- ========================================== -->  <filter>    <description>字符集过滤器</description>    <filter-name>characterEncodingFilter</filter-name>    <filter-class>org.springframework.web.filter.CharacterEncodingFilter</filter-class>    <init-param>      <description>字符集编码</description>      <param-name>encoding</param-name>      <param-value>UTF-8</param-value>    </init-param>    <init-param>      <param-name>forceEncoding</param-name>      <param-value>true</param-value>    </init-param>  </filter>  <filter-mapping>    <filter-name>characterEncodingFilter</filter-name>    <url-pattern>/*</url-pattern>  </filter-mapping>  <!-- ====================== -->  <!-- SpringMVC配置 -->  <!-- ====================== -->  <servlet>    <servlet-name>springmvc</servlet-name>    <servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>    <init-param>      <description>spring mvc 配置文件</description>      <param-name>contextConfigLocation</param-name>      <param-value>classpath*:/servlet-context.xml</param-value>    </init-param>    <load-on-startup>1</load-on-startup>  </servlet>  <servlet-mapping>    <servlet-name>springmvc</servlet-name>    <url-pattern>/</url-pattern>  </servlet-mapping>  <welcome-file-list>    <welcome-file>/index.jsp</welcome-file>  </welcome-file-list></web-app>

servlet-context.xml

<?xml version="1.0" encoding="UTF-8"?><beans xmlns="http://www.springframework.org/schema/beans"   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"   xmlns:context="http://www.springframework.org/schema/context"   xmlns:aop="http://www.springframework.org/schema/aop"   xmlns:mvc="http://www.springframework.org/schema/mvc"   xmlns:tx="http://www.springframework.org/schema/tx"   xmlns:p="http://www.springframework.org/schema/p"   xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.1.xsdhttp://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-4.1.xsdhttp://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-4.1.xsdhttp://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-4.1.xsdhttp://www.springframework.org/schema/mvc http://www.springframework.org/schema/mvc/spring-mvc-4.1.xsd"   default-autowire="byName"><description>Spring MVC视图解析配置</description><import resource="classpath:spring/hive-config.xml" /><!-- ======================================= --><!-- spring组件扫描 --><!-- ======================================= --><context:component-scan base-package="com.jikefriend.test.hdfs" /><!--对静态资源文件的访问--><!--<mvc:resources mapping="/resource/**" location="/resource/" />--><mvc:annotation-driven><mvc:message-converters><bean id="mappingJacksonHttpMessageConverter" class="org.springframework.http.converter.json.MappingJackson2HttpMessageConverter"><property name="supportedMediaTypes"><list><value>application/json;charset=UTF-8</value><value>application/x-www-form-urlencoded;charset=UTF-8</value></list></property></bean><bean class="org.springframework.http.converter.StringHttpMessageConverter" /><bean class="org.springframework.http.converter.FormHttpMessageConverter" /><bean class="org.springframework.http.converter.BufferedImageHttpMessageConverter" /><bean class="org.springframework.http.converter.ByteArrayHttpMessageConverter" /><bean class="org.springframework.http.converter.ResourceHttpMessageConverter" /></mvc:message-converters></mvc:annotation-driven></beans>

Hive相关配置

hive-config.xml

<?xml version="1.0" encoding="UTF-8"?><beans:beans xmlns="http://www.springframework.org/schema/hadoop"             xmlns:beans="http://www.springframework.org/schema/beans"             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"             xmlns:context="http://www.springframework.org/schema/context"             xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd  http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd  http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd">    <context:property-placeholder location="classpath*:**/*.properties"/>    <context:component-scan base-package="com.jikefriend.test.hdfs.hive" />    <configuration>        fs.defaultFS=${hd.fs}    </configuration>    <!-- This sample requires a running HiveServer2 -->    <hive-client-factory id="hiveClientFactory" hive-data-source-ref="hiveDataSource"/>    <beans:bean id="hiveDriver" class="org.apache.hive.jdbc.HiveDriver"/>    <beans:bean id="hiveDataSource" class="org.springframework.jdbc.datasource.SimpleDriverDataSource">        <beans:constructor-arg name="driver" ref="hiveDriver"/>        <beans:constructor-arg name="url" value="${hive.url}"/>    </beans:bean>    <hive-template id="hiveTemplate"/></beans:beans>

hive.properties

hive.host=example.hive.hosthive.port=10000hive.url=jdbc:hive2://${hive.host}:${hive.port}hd.fs=hdfs://example.hive.host:8020hd.rm=example.hive.host:8032hd.jh=example.hive.host:10020

代码编写

与hive交互的Repository.java

package com.jikefriend.test.hdfs.hive;import com.google.common.base.Strings;import org.springframework.data.hadoop.hive.HiveClient;import org.springframework.data.hadoop.hive.HiveClientFactory;import org.springframework.data.hadoop.hive.HiveTemplate;import org.springframework.stereotype.Component;import javax.annotation.Resource;import java.sql.SQLException;import java.util.List;@Componentpublic class TableRepository {    @Resource    private HiveTemplate hiveTemplate;    @Resource    private HiveClientFactory hiveClientFactory;    public List findAllTables() {        return hiveTemplate.query("show tables");    }    public Long count(String tableName) {        if (Strings.isNullOrEmpty(tableName))            return -1L;        HiveClient hiveClient = hiveClientFactory.getHiveClient();        try {            String result = hiveClient.executeAndfetchOne("select count(*) from " + tableName);            return Long.parseLong(result);        } finally {            try {                hiveClient.shutdown();            } catch (SQLException e) {                e.printStackTrace();            }        }    }}

到此就算结束了，可以通过hive shell来进行验证

数据导入Hive

与导入Hbase一样，采用sqoop，命令也大体一致，如下所示

sqoop import --connect jdbc:mysql://mysql.host:mysqlport/databasename \  --table [mysqltable] --hive-table [hbasetable]  \  --username [mysqluname] --password [mysqlpwd]