Spring中使用Spark连接的DataSource

来源：互联网发布：福禄克网络sn查询编辑：程序博客网时间：2024/05/18 01:27

在Spring中配置Spark hive-thriftserver的连接DataSource与配置其他数据源连接方式是一样的，如一般Oracle数据源配置，使用如下的Jar包列表

使用jdbc的方式连接thriftserver服务访问表数据

package com.hadoop.test; import java.sql.Connection;import java.sql.DriverManager;import java.sql.PreparedStatement;import java.sql.ResultSet;import java.sql.SQLException;import java.sql.Statement; public class ThreadSql extends  Thread  {     @Override    public void run() {                 excSparkSql();    }         public  void  excSparkSql() {        String result = "";        Connection conn = null;        Statement stmt = null;        try {            Class.forName("org.apache.hadoop.hive.jdbc.HiveDriver");            conn = DriverManager.getConnection(                    "jdbc:hive://10.20.12.214:10000/test_db", "hduser001", "");            stmt = conn.createStatement();            String sql = " select count(1) from test_data ";               long start = System.currentTimeMillis();                         ResultSet res = stmt.executeQuery(sql);                         long cost = System.currentTimeMillis() - start;                         result = Thread.currentThread().getName() + ": " + cost/1000.0f + "s";            System.out.println(result);            stmt.close();            conn.close();        } catch (Exception e) {                         try {                stmt.close();                conn.close();            } catch (SQLException e1) {                // TODO Auto-generated catch block                e1.printStackTrace();            }            e.printStackTrace();                     }             }      }

在Spring中配置类似oracle的datasource配置，并在其他的ServiceBean中将其注入即可

    <bean id="dataService" class="com.sun.dt.service.impl.DataServiceImpl">        <property name="dataDao" ref="sparkDataSource"></property>    </bean>         <bean id="oralceDataSource" class="org.apache.commons.dbcp.BasicDataSource"           destroy-method="close">          <property name="driverClassName" value="${jdbc.driver}" />          <property name="url" value="${jdbc.url}" />          <property name="username" value="${jdbc.username}" />          <property name="password" value="${jdbc.password}" />        <property name="maxActive" value="${jdbc.maxActive}" />        <property name="maxIdle" value="${jdbc.maxIdle}" />        <property name="maxWait" value="${jdbc.maxWait}" />        <property name="defaultAutoCommit" value="${jdbc.defaultAutoCommit}" />    </bean>                <bean id="sparkDataSource" class="org.apache.commons.dbcp.BasicDataSource"           destroy-method="close">          <property name="driverClassName" value="org.apache.hadoop.hive.jdbc.HiveDriver" />          <property name="url" value="jdbc:hive://192.168.10.124:10000/test_db" />          <property name="username" value="hduser001" />          <property name="password" value="" />        <property name="maxActive" value="8" />        <property name="maxIdle" value="3" />        <property name="maxWait" value="5" />    </bean>

0 0