大数据开发例子
来源：互联网 发布：专业调色软件app 编辑：程序博客网 时间：2024/04/30 10:57
import java.io.IOExceptionimport java.sql.Timestampimport java.util.Dateimport DAO.{ScalaConn, ScalaHbase}import org.apache.commons.lang3.time.DateFormatUtilsimport org.apache.hadoop.conf.Configurationimport org.apache.hadoop.hbase.client._import org.apache.hadoop.hbase.io.ImmutableBytesWritableimport org.apache.hadoop.hbase.mapreduce.TableInputFormatimport org.apache.hadoop.hbase.util.Bytesimport org.apache.hadoop.hbase.{MasterNotRunningException, TableName, ZooKeeperConnectionException}import org.apache.spark.sql.DataFrameimport org.slf4j.LoggerFactory/** 新手标已还款追踪数据。* author：谭志坚* */@SerialVersionUID(-1)object dw_yhk_zz{ def LOG = LoggerFactory.getLogger(getClass) @throws(classOf[Exception]) def dw_yhk_zz_run( conf: Configuration, hconn: Connection){ val familyColumn: Array[String] = Array[String]("USERS","INFO") ScalaHbase.createTable(conf,hconn, "DW_YHK_ZZ", familyColumn) ScalaHbase.createTable(conf,hconn, "DW_YHK_WCZ", familyColumn) ScalaHbase.createTable(conf,hconn, "DW_YHK_XT", familyColumn) val spark = ScalaConn.spark val DW_YHK_ZZ: Table = hconn.getTable(TableName.valueOf("DW_YHK_ZZ")) val DW_YHK_WCZ: Table = hconn.getTable(TableName.valueOf("DW_YHK_WCZ")) val DW_YHK_XT: Table = hconn.getTable(TableName.valueOf("DW_YHK_XT")) var maxdate: String = ScalaConn.getMaxDate(conf,hconn, DW_YHK_ZZ, "USERS", "DT")// var DW_YHK_WCZ_maxdate: String = ScalaConn.getMaxDate(conf,hconn, DW_YHK_WCZ, "USERS", "DT")// maxdate = "2017-01-01 00:00:00"// val date: Date = new Date();// var maxdt: Timestamp = new Timestamp(date.getTime) ; var maxdt:String= String.valueOf(ScalaConn.getYesterday(-7,"yyyy-MM-dd HH:mm:ss"))// if (maxdate<maxdt){// maxdt = maxdate// }// if (maxdate.length < 10){// maxdate = maxdate + "000000"// maxdt = ScalaConn.GetStampByTime1(maxdate)// }else {// maxdate = maxdate.substring(0, 10) + " 00:00:00"// maxdt = ScalaConn.GetStampByTime(maxdate)// } val table_planing_bill:String ="t_planing_bill_invests" val table_fxgo_bill:String ="t_fxgo_bill_invests" val table_bill:String ="t_bill_invests" val table_bill_huoqibao:String ="t_huoqibao_user_trade_record" val tableInvests: String = "t_invests" val tableBids: String = "t_bids" val tableBidIn:String="t_bid_increase_interest" val tableUsers: String = "t_users" val tablFxgo: String = "t_fxgo_invests" val tablTtz:String ="t_huoqibao_user_invest" val tablePinvest: String = "t_planing_invests" val tablePlanings: String = "t_planing" val tableDetails: String = "t_user_details" val DFplaning_bill: DataFrame = ScalaConn.getP2PDataFrame(table_planing_bill) val DFfxgo_bill: DataFrame = ScalaConn.getP2PDataFrame(table_fxgo_bill) val DFbill: DataFrame = ScalaConn.getP2PDataFrame(table_bill) val DFbill_huoqibao: DataFrame = ScalaConn.getP2PDataFrame(table_bill_huoqibao) val DFInvests: DataFrame = ScalaConn.getP2PDataFrame(tableInvests) val DFBids: DataFrame = ScalaConn.getP2PDataFrame(tableBids) val DFBidIn: DataFrame = ScalaConn.getP2PDataFrame(tableBidIn) val DFUsers: DataFrame = ScalaConn.getP2PDataFrame(tableUsers) val DFFxgo: DataFrame = ScalaConn.getP2PDataFrame(tablFxgo) val DFTtz:DataFrame = ScalaConn.getP2PDataFrame(tablTtz) val DFPinvest: DataFrame = ScalaConn.getP2PDataFrame(tablePinvest) val DFPlanings: DataFrame = ScalaConn.getP2PDataFrame(tablePlanings) val DFDetails: DataFrame = ScalaConn.getP2PDataFrame(tableDetails) DFplaning_bill.createOrReplaceTempView("t_planing_bill_invests") DFfxgo_bill.createOrReplaceTempView("t_fxgo_bill_invests") DFbill.createOrReplaceTempView("t_bill_invests") DFbill_huoqibao.createOrReplaceTempView("t_huoqibao_user_trade_record") DFInvests.createOrReplaceTempView("t_invests") DFBids.createOrReplaceTempView("t_bids") DFBidIn.createOrReplaceTempView("t_bid_increase_interest") DFUsers.createOrReplaceTempView("t_users") DFFxgo.createOrReplaceTempView("t_fxgo_invests") DFTtz.createOrReplaceTempView("t_huoqibao_user_invest") DFPinvest.createOrReplaceTempView("t_planing_invests") DFPlanings.createOrReplaceTempView("t_planing") DFDetails.createOrReplaceTempView("t_user_details") conf.set(TableInputFormat.INPUT_TABLE, "DW_YHK_ZZ") import spark.implicits._ var hbaseRDD = spark.sparkContext.newAPIHadoopRDD(conf, classOf[TableInputFormat], classOf[ImmutableBytesWritable], classOf[Result]) var HbaseDF = hbaseRDD.map(r=>( Bytes.toString(r._2.getRow), Bytes.toString(r._2.getValue(Bytes.toBytes("USERS"),Bytes.toBytes("USER_ID"))), Bytes.toString(r._2.getValue(Bytes.toBytes("USERS"),Bytes.toBytes("REG_TIME"))), Bytes.toString(r._2.getValue(Bytes.toBytes("USERS"),Bytes.toBytes("DT"))), Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("RECHARGE"))), Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("HKAMT"))), Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("HKDT"))), Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("PERIODS"))), Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("WITHDRAW"))), Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("FJH"))), Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("NEW"))), Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("XDT"))), Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("TTZ"))), Bytes.toString(r._2.getValue(Bytes.toBytes("INFO"),Bytes.toBytes("FXG"))) )).toDF("ROWKEY","USER_ID","REG_TIME","DT","RECHARGE","HKAMT","HKDT","PERIODS","WITHDRAW","FJH","NEW","XDT","TTZ","FXG") HbaseDF.createOrReplaceTempView("DW_YHK_ZZ") val userSql = "SELECT\n\ta.bid_id,\n\ta.user_id,\n\ta. NAME,\n\ta.reality_name,\n\ta.mobile,\n\ta.time,\n\ta.special_invitation_code,\n\ta.title,\n\ta.periods,\n\ta.hk_amt,\n\ta.hk_dt,\n\tc.withdraw,\n\tc.dt,\n\tc.periods,\n\tc.plan_amount,\n\tc.new_amount,\n\tc.xdt_amount,\n\tc.ttz_amount,\n\tc.fxgo_amount\nFROM\n\t(\n\t\tSELECT\n\t\t\ttpi.user_id,\n\t\t\ttp.cycle_days periods,\n\t\t\ttpi.time dt,\n\t\t\tSUM(tpi.amount) plan_amount,\n\t\t\t'0' new_amount,\n\t\t\t'0' xdt_amount,\n\t\t\t'0' ttz_amount,\n\t\t\t'0' fxgo_amount,\t\t\t\n\t\t\t'0' withdraw\n\t\tFROM\n\t\t\tt_planing_invests tpi\n\t\tJOIN t_planing tp ON tpi.planing_id = tp.id\n\t\tWHERE\n\t\t\ttpi.transfer_id = 0\n\t\tAND tpi.time >= '"+maxdt+"'\n\t\tGROUP BY\n\t\t\ttpi.user_id,\n\t\t\ttp.cycle_days,\n\t\t\ttpi.time\n\t\tUNION ALL\n\t\t\tSELECT\n\t\t\t\tts.user_id,\n\t\t\t\t(\n\t\t\t\t\tCASE\n\t\t\t\t\tWHEN tb.period_unit = 0 THEN\n\t\t\t\t\t tb.period *30\n\t\t\t\t\tWHEN tb.period_unit = 1 THEN\n\t\t\t\t\t\ttb.period\n\t\t\t\t\tWHEN tb.period_unit = - 1 THEN\n\t\t\t\t\t\ttb.period *12 *30\n\t\t\t\t\tEND\n\t\t\t\t) AS periods,\n\t\t\t\tts.time dt,\n\t\t\t\t'0' plan_amount,\n\t\t\t\tsum(\n\n\t\t\t\t\tIF (tnew.bid_id > 0, ts.amount, 0)\n\t\t\t\t) new_amount,\n\t\t\t\tsum(\n\n\t\t\t\t\tIF (tnew.bid_id > 0, 0, ts.amount)\n\t\t\t\t) xdt_amount,\n\t\t\t\t'0' ttz_amount,\n\t\t\t\t'0' fxgo_amount,\t\t\t\n\t\t\t\t'0' withdraw\n\t\t\tFROM\n\t\t\t\tt_invests ts\n\t\t\tJOIN t_bids tb ON ts.bid_id = tb.id\n\t\t\tLEFT JOIN (\n\t\t\t\tSELECT\n\t\t\t\t\tbid_id\n\t\t\t\tFROM\n\t\t\t\t\tt_bid_increase_interest\n\t\t\t\tWHERE\n\t\t\t\t\tis_novice = 1\n\t\t\t) tnew ON ts.bid_id = tnew.bid_id\n\t\t\tWHERE\n\t\t\t\tts.transfers_id = 0\n\t\t\tAND ts.time >= '"+maxdt+"'\n\t\t\tGROUP BY\n\t\t\t\tts.user_id,\n\t\t\t\ttb.period_unit,\n\t\t\t\ttb.period,\n\t\t\t\tts.time\n\t\t\tUNION ALL\n\t\t\t\tSELECT\n\t\t\t\t\tuser_id,\n\t\t\t\t\t'1' periods,\n\t\t\t\t\tapply_date dt,\n\t\t\t\t\t'0' plan_amount,\n\t\t\t\t\t'0' new_amount,\n\t\t\t\t\t'0' xdt_amount,\n\t\t\t\t\tsum(apply_amount) ttz_amount,\n\t\t\t\t\t'0' fxgo_amount,\t\t\t\t\t\n\t\t\t\t\t'0' withdraw\n\t\t\t\tFROM\n\t\t\t\t\tt_huoqibao_user_invest\n\t\t\t\tWHERE\n\t\t\t\t\tSTATUS = 1\n\t\t\t\tAND apply_date >= '"+maxdt+"'\n\t\t\t\tGROUP BY\n\t\t\t\t\tuser_id,\n\t\t\t\t\tapply_date\n\t\t\t\tUNION ALL\n\t\t\t\t\tSELECT\n\t\t\t\t\t\tts.user_id,\n\t\t\t\t\t\tts.periods * 30 periods,\n\t\t\t\t\t\tts.time dt,\n\t\t\t\t\t\t'0' plan_amount,\n\t\t\t\t\t\t'0' new_amount,\n\t\t\t\t\t\t'0' xdt_amount,\n\t\t\t\t\t\t'0' ttz_amount,\n\t\t\t\t\t\tsum(ts.amount) fxgo_amount,\t\t\t\t\t\t\n\t\t\t\t\t\t'0' withdraw\n\t\t\t\t\tFROM\n\t\t\t\t\t\tt_fxgo_invests ts\n\t\t\t\t\tWHERE\n\t\t\t\t\t\tts. STATUS = 1\n\t\t\t\t\tAND ts.time >= '"+maxdt+"'\n\t\t\t\t\tGROUP BY\n\t\t\t\t\t\tts.user_id,\n\t\t\t\t\t\tts.periods,\n\t\t\t\t\t\tts.time\n\t\t\t\t\tUNION ALL\n\t\t\t\t\t\tSELECT\n\t\t\t\t\t\t\tuser_id,\n\t\t\t\t\t\t\t'0' periods,\n\t\t\t\t\t\t\ttime dt,\n\t\t\t\t\t\t\t'0' plan_amount,\n\t\t\t\t\t\t\t'0' new_amount,\n\t\t\t\t\t\t\t'0' xdt_amount,\n\t\t\t\t\t\t\t'0' ttz_amount,\n\t\t\t\t\t\t\t'0' fxgo_amount,\t\t\t\t\t\t\n\t\t\t\t\t\t\tsum(\n\n\t\t\t\t\t\t\t\tIF (\n\t\t\t\t\t\t\t\t\toperation = 304\n\t\t\t\t\t\t\t\t\tOR operation = 379,\n\t\t\t\t\t\t\t\t\tIFNULL(amount, 0),\n\t\t\t\t\t\t\t\t\t0\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\t\t\t) withdraw\n\t\t\t\t\t\tFROM\n\t\t\t\t\t\t\tt_user_details\n\t\t\t\t\t\tWHERE\n\t\t\t\t\t\t\toperation IN (304, 379)\n\t\t\t\t\t\tAND time >= '"+maxdt+"'\n\t\t\t\t\t\tGROUP BY\n\t\t\t\t\t\t\tuser_id,\n\t\t\t\t\t\t\ttime\n\t) c\nRIGHT JOIN (\n\tSELECT\n\t\ttb.user_id,\n\t\tu. NAME,\n\t\tu.reality_name,\n\t\tu.mobile,\n\t\tu.time,\n\t\tu.special_invitation_code,\n\t\ttb.bid_id,\n\t\ttb.title,\n\t\tIFNULL(bid.period, 0) AS periods,\n\t\ttb.real_receive_time hk_dt,\t\t\n\t\tsum(\n\t\t\ttb.real_receive_corpus + tb.real_receive_interest\n\t\t) AS hk_amt\n\tFROM\t\t\n\t\t\t\tt_bill_invests tb\n\t\tJOIN (\n\t\t\t\tSELECT\n\t\t\t\t\tbid_id\n\t\t\t\tFROM\n\t\t\t\t\tt_bid_increase_interest\n\t\t\t\tWHERE\n\t\t\t\t\tis_novice = 1\n\t\t\t) tnew ON tb.bid_id = tnew.bid_id\t\t\t\n\tJOIN t_users u ON tb.user_id = u.id\n JOIN t_bids bid ON bid.id = tb.bid_id\n\tWHERE tb.real_receive_time >= '"+maxdt+"'\t\n and tb.user_id not in (63,16721,130845,9944,75,9677,163,9607,16648,16668)\n and tb.real_receive_corpus+tb.real_receive_interest >0\n\tGROUP BY\n\t\ttb.user_id,\n\t\tu. NAME,\n\t\tu.reality_name,\n\t\tu.mobile,\n\t\tu.time,\n\t\tu.special_invitation_code,\n\t\ttb.bid_id,\n\t\ttb.title,\n\t\tbid.period,\n\t\ttb.real_receive_time\t\n) a ON a.user_id = c.user_id\nWHERE a.hk_dt <= c.dt" val DW_YHK_WCZ_userSql ="select wcz.user_id,sum(wcz.HKAMT) HKAMT,sum(wcz.invtamt) invtamt,sum(wcz.WITHDRAW) WITHDRAW, sum(wcz.wcz_amt) wcz_amt from (\nSELECT w3.user_id,sum(to_number(w3.HKAMT)) HKAMT,0 invtamt,0 WITHDRAW,sum(to_number(w3.HKAMT)) wcz_amt from (\nselect t1.USER_ID,t1.HKAMT from (\nselect DISTINCT dyz2.USER_ID,dyz2.HKAMT,dyz2.HKDT, dyz2.DT,dyz2.PERIODS,dyz2.WITHDRAW,dyz2.FJH,dyz2.NEW,dyz2.XDT,dyz2.TTZ,dyz2.FXG from DW_YHK_ZZ dyz2 where dyz2.HKDT >='"+maxdt+"'\n)t1 WHERE T1.DT IS NULL\n) w3 group by w3.user_id\nunion all\nSELECT w1.user_id user_id,w1.HKAMT HKAMT,w2.invtamt invtamt,w2.WITHDRAW WITHDRAW,w1.HKAMT-w2.invtamt-w2.WITHDRAW AS wcz_amt from \n(SELECT t3.user_id,sum(to_number(t3.HKAMT)) HKAMT from (\nselect distinct dyz3.USER_ID,dyz3.REG_TIME,dyz3.HKAMT,dyz3.HKDT from DW_YHK_ZZ dyz3 \nwhere dyz3.HKDT >='"+maxdt+"' \n)t3 GROUP BY t3.user_id) w1 join \n(select t2.USER_ID,sum(to_number(t2.WITHDRAW)) WITHDRAW,sum(to_number(t2.FJH)+to_number(t2.NEW)+to_number(t2.XDT)+to_number(t2.TTZ)+to_number(t2.FXG)) invtamt from (\nselect distinct dyz2.USER_ID,dyz2.DT,dyz2.PERIODS,dyz2.WITHDRAW,dyz2.FJH,dyz2.NEW,dyz2.XDT,dyz2.TTZ,dyz2.FXG from DW_YHK_ZZ dyz2 \nwhere dyz2.HKDT >='"+maxdt+"' and to_number(dyz2.periods)>=0 \n)t2 group by t2.USER_ID) w2 on w1.user_id = w2.USER_ID WHERE w1.HKAMT>(w2.invtamt+w2.WITHDRAW)\n) wcz group by wcz.user_id" val DW_YHK_XT_userSql ="SELECT w1.user_id,w1.HKAMT,w2.PERIODS,w2.WITHDRAW,w2.FJH,w2.NEW,w2.XDT,w2.TTZ,w2.FXG from \n(\nSELECT t3.user_id,sum(to_number(t3.HKAMT)) HKAMT from (\nselect distinct dyz3.USER_ID,dyz3.REG_TIME,dyz3.HKAMT,dyz3.HKDT from DW_YHK_ZZ dyz3 \nwhere dyz3.HKDT >='"+maxdt+"' \n)t3 GROUP BY t3.user_id) w1 \nLEFT JOIN\n(\nselect t2.USER_ID,t2.PERIODS,sum(to_number(t2.WITHDRAW)) WITHDRAW,sum(to_number(t2.FJH)) FJH,SUM(to_number(t2.NEW)) NEW,SUM(to_number(t2.XDT)) XDT,SUM(to_number(t2.TTZ)) TTZ,SUM(to_number(t2.FXG)) FXG from \n(select distinct dyz2.USER_ID,dyz2.DT,dyz2.PERIODS,dyz2.WITHDRAW,dyz2.FJH,dyz2.NEW,dyz2.XDT,dyz2.TTZ,dyz2.FXG from DW_YHK_ZZ dyz2 \nwhere dyz2.HKDT >='"+maxdt+"' and to_number(dyz2.periods)>=0 \n)t2 group by t2.USER_ID,t2.PERIODS\n) w2 on w1.user_id = w2.USER_ID" var userDF: DataFrame = spark.sql(userSql) try { userDF.collect().foreach { userRow => { val bid_id: String = String.valueOf(userRow.getLong(0)) val user_id: String = String.valueOf(userRow.getLong(1)) val name: String = String.valueOf(userRow.getString(2)) val reality_name: String = String.valueOf(userRow.getString(3)) val mobile: String = String.valueOf(userRow.getString(4)) val reg_time: String = String.valueOf(userRow.getTimestamp(5)) val special_invitation_code: String = String.valueOf(userRow.getString(6)) val title: String = String.valueOf(userRow.getString(7)) val bid_periods: String = String.valueOf(userRow.getInt(8)) val balance: String = String.valueOf(userRow.getDecimal(9)) val hk_amt: String = String.valueOf(userRow.getDecimal(10)) val hk_dt: String = String.valueOf(userRow.getTimestamp(11)) val ts2:Timestamp =ScalaConn.GetStampByTime(hk_dt) val recharge: String = String.valueOf(userRow.getString(12)) val withdraw: String = String.valueOf(userRow.getString(13)) var dt: String ="" var Rkey2: String="" val value14 = userRow.isNullAt(14) if (value14 != true) { dt = String.valueOf(userRow.getTimestamp(14)) val ts:Timestamp =ScalaConn.GetStampByTime(dt) Rkey2 = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMddHHmmss")).toString }// val dt: String = String.valueOf(userRow.getTimestamp(14)) var periods: String = "" val value9 = userRow.isNullAt(15) if (value9 != true) { periods = String.valueOf(userRow.getString(15)) }// val periods: String = String.valueOf(userRow.getDouble(14)) val plan_amount: String = String.valueOf(userRow.getString(16)) val new_amount: String = String.valueOf(userRow.getString(17)) val xdt_amount: String = String.valueOf(userRow.getString(18)) val ttz_amount: String = String.valueOf(userRow.getString(19)) val fxgo_amount: String = String.valueOf(userRow.getString(20)) // val Rkey: String = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMdd HH:mm:ssHH")).toString val Rkey1: String = new StringBuffer(DateFormatUtils.format(ts2, "yyyyMMddHHmmss")).toString var rowKey: String=Rkey1 val id:String = String.format(String.format("%010d",Integer.valueOf(user_id))) if (Rkey2 != "") { rowKey= rowKey + Rkey2 + id }else{ rowKey= rowKey + rowKey + id } ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "USER_ID", user_id) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "NAME", name) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "REALITY_NAME", reality_name) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "MOBILE", mobile) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "REG_TIME", reg_time) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "SPECIAL_INVITATION_CODE", special_invitation_code) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "BALANCE", balance) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "DT", dt) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "TITLE", title) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "USERS", "BIDPERIODS", bid_periods) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "HKAMT", hk_amt) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "HKDT", hk_dt) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "PERIODS", periods) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "RECHARGE", recharge) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "WITHDRAW", withdraw) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "FJH", plan_amount) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "NEW", new_amount) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "XDT", xdt_amount) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "TTZ", ttz_amount) ScalaHbase.addRow(DW_YHK_ZZ, rowKey, "INFO", "FXG", fxgo_amount) } } } catch { case e: Exception => { if (e.getClass == classOf[MasterNotRunningException]) { LOG.error("MasterNotRunningException:"+e.toString) } if (e.getClass == classOf[ZooKeeperConnectionException]) { LOG.error("ZooKeeperConnectionException:"+e.toString) } if (e.getClass == classOf[IOException]) { LOG.error("IOException:"+e.toString) } LOG.error("error xception:"+ e.printStackTrace) } } finally { userDF = null DW_YHK_ZZ.close } var DW_YHK_WCZDF: DataFrame = spark.sql(DW_YHK_WCZ_userSql) try { DW_YHK_WCZDF.collect().foreach { userRow => { val user_id: String = String.valueOf(userRow.getString(0)) val HKAMT: String = String.valueOf(userRow.getString(1)) val invtamt: String = String.valueOf(userRow.getDouble(2)) val WITHDRAW: String = String.valueOf(userRow.getDouble(3)) val wcz_amt: String = String.valueOf(userRow.getDouble(4))// ("ROWKEY","USER_ID","REG_TIME","DT","RECHARGE","HKAMT","HKDT","PERIODS","WITHDRAW","FJH","NEW","XDT","TTZ","FXG") // val ts: Timestamp = ScalaConn.GetStampByTime(ymd) val date: Date = new Date(); var ts: Timestamp = new Timestamp(date.getTime) ; ts = ScalaConn.GetStampByTime(maxdt)// val rowKey: String = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMdd")).toString + DateFormatUtils.format(ts, "HHmmss") val Rkey1: String = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMddHHmmss")).toString val id:String = String.format("%010d",Integer.valueOf(user_id)) val rowKey = Rkey1 + id // val rowKey: String = ScalaConn.GetStampByTime1(ymd).toString ScalaHbase.addRow(DW_YHK_WCZ, rowKey, "USERS", "USER_ID", user_id) ScalaHbase.addRow(DW_YHK_WCZ, rowKey, "USERS", "HKAMT", HKAMT) ScalaHbase.addRow(DW_YHK_WCZ, rowKey, "USERS", "INVTAMT", invtamt) ScalaHbase.addRow(DW_YHK_WCZ, rowKey, "USERS", "WITHDRAW", WITHDRAW) ScalaHbase.addRow(DW_YHK_WCZ, rowKey, "USERS", "WCZ_AMT", wcz_amt) } } } catch { case e: Exception => { if (e.getClass == classOf[MasterNotRunningException]) { LOG.error("MasterNotRunningException:" + e.toString) } if (e.getClass == classOf[ZooKeeperConnectionException]) { LOG.error("ZooKeeperConnectionException:" + e.toString) } if (e.getClass == classOf[IOException]) { LOG.error("IOException:" + e.toString) } e.printStackTrace } } finally { DW_YHK_WCZDF = null if (null != DW_YHK_WCZ) { DW_YHK_WCZ.close } }// w1.user_id,w1.HKAMT,w2.PERIODS,w2.WITHDRAW,w2.FJH,w2.NEW,w2.XDT,w2.TTZ,w2.FXG var DW_YHK_XTDF: DataFrame = spark.sql(DW_YHK_XT_userSql) try { DW_YHK_XTDF.collect().foreach { userRow => { val user_id: String = String.valueOf(userRow.getString(0)) val HKAMT: String = String.valueOf(userRow.getDouble(1)) val PERIODS: String = String.valueOf(userRow.getString(2)) val WITHDRAW: String = String.valueOf(userRow.getDouble(3)) val FJH: String = String.valueOf(userRow.getDouble(4)) val NEW: String = String.valueOf(userRow.getDouble(5)) val XDT: String = String.valueOf(userRow.getDouble(6)) val TTZ: String = String.valueOf(userRow.getDouble(7)) val FXG: String = String.valueOf(userRow.getDouble(8)) // ("ROWKEY","USER_ID","REG_TIME","DT","RECHARGE","HKAMT","HKDT","PERIODS","WITHDRAW","FJH","NEW","XDT","TTZ","FXG") // val ts: Timestamp = ScalaConn.GetStampByTime(ymd) val date: Date = new Date(); var ts: Timestamp = new Timestamp(date.getTime) ; ts = ScalaConn.GetStampByTime(maxdt) // val rowKey: String = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMdd")).toString + DateFormatUtils.format(ts, "HHmmss") val Rkey1: String = new StringBuffer(DateFormatUtils.format(ts, "yyyyMMddHHmmss")).toString val id:String = String.format("%010d",Integer.valueOf(user_id)) val rowKey = Rkey1 + id+PERIODS // val rowKey: String = ScalaConn.GetStampByTime1(ymd).toString ScalaHbase.addRow(DW_YHK_XT, rowKey, "USERS", "USER_ID", user_id) ScalaHbase.addRow(DW_YHK_XT, rowKey, "USERS", "HKAMT", HKAMT) ScalaHbase.addRow(DW_YHK_XT, rowKey, "USERS", "PERIODS", PERIODS) ScalaHbase.addRow(DW_YHK_XT, rowKey, "USERS", "WITHDRAW", WITHDRAW) ScalaHbase.addRow(DW_YHK_XT, rowKey, "USERS", "FJH", FJH) ScalaHbase.addRow(DW_YHK_XT, rowKey, "USERS", "NEW", NEW) ScalaHbase.addRow(DW_YHK_XT, rowKey, "USERS", "XDT", XDT) ScalaHbase.addRow(DW_YHK_XT, rowKey, "USERS", "TTZ", TTZ) ScalaHbase.addRow(DW_YHK_XT, rowKey, "USERS", "FXG", FXG) } } } catch { case e: Exception => { if (e.getClass == classOf[MasterNotRunningException]) { LOG.error("MasterNotRunningException:" + e.toString) } if (e.getClass == classOf[ZooKeeperConnectionException]) { LOG.error("ZooKeeperConnectionException:" + e.toString) } if (e.getClass == classOf[IOException]) { LOG.error("IOException:" + e.toString) } e.printStackTrace } } finally { DW_YHK_XTDF = null if (null != DW_YHK_WCZ) { DW_YHK_XT.close } } }}
0 0
- 大数据开发例子
- 大数据 IMF传奇行动 hadoop 中 开发mapreduce 天气预报的例子
- 大数据开发视频
- 大数据-开发利器
- 大数据开发-01
- 大数据开发进阶
- 大数据-开发利器
- 大数据开发初探
- 大数据-开发利器
- 大数据开发工具
- Hadoop-MapReduce-TeraSort-大数据排序例子
- 大数据读写分离实现的例子
- 大数据日志分析小例子
- 大数据应用的几个典型例子
- 大数据平台开发高级工程师
- 对大数据开发经验
- 大数据开发必备工具
- 趋势分析：大数据开发
- 八大排序算法
- 二叉树的相关算法
- 简单易学的机器学习算法——Softmax Regression
- jvm内存区域复习
- RFS的web自动化验收测试——第1讲 菜单栏&工具栏
- 大数据开发例子
- 动态规划—26,30滑雪
- Apache 80端口被占用
- TypeScript基本数据类型
- 使用HttpClient实现文件的上传
- org.springframework.beans.factory.xml.XmlBeanDefinitionStoreException: Line 23 in XML document from
- HDU 5738 Eureka
- python实现的简易员工管理系统
- 数据库
