spark 实战笔记case3

val mapper = sc.textFile("/user/hive/warehouse/sla.db/e*_30days/").map(line => line.split("\073")).map(line => ( if (line(0)=="598017" || line(0)=="614420" || line(0)=="614421") line(5) else line(4), ( line(0), line(2).substring(0, 19) ) ))def filterSLA ( events:Seq[(String,String)] ): Boolean = {  val eventMap = events.sortBy(_._2).toMap  if (eventMap contains "614401") {    val format = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss")    val ts_614401 = format.parse(eventMap("614401")).getTime    if (eventMap contains "401411") {      val ts_401411 = format.parse(eventMap("401411")).getTime      if (ts_401411 - ts_614401 > 10*60*1000) return true    }    if (eventMap contains "393217") {      val ts_393217 = format.parse(eventMap("393217")).getTime      if (ts_393217 - ts_614401 > 60*60*1000) return true    }    if (eventMap contains "614405") {      val ts_614405 = format.parse(eventMap("614405")).getTime      if (ts_614405 - ts_614401 > 48*60*60*1000) return true    }    return false  } else {    return false  }}def flatValues ( events:Seq[(String,String)] ): String = {  val eventMap = events.sortBy(_._2).toMap  return (if (eventMap contains "614401") eventMap("614401") else "NULL") + "\t" + (if (eventMap contains "401411") eventMap("401411") else "NULL") + "\t" + (if (eventMap contains "393217") eventMap("393217") else "NULL") + "\t" + (if (eventMap contains "614405") eventMap("614405") else "NULL")}mapper.groupByKey.filter( kv => filterSLA(kv._2.toSeq) ).map( kv => kv._1 + "\t" + flatValues(kv._2.toSeq) ).saveAsTextFile("/tmp/opt")

import org.apache.hadoop.hbase.{HBaseConfiguration, HTableDescriptor}import org.apache.hadoop.hbase.mapreduce.TableInputFormatimport org.apache.spark.rdd.NewHadoopRDDval conf = HBaseConfiguration.create()conf.set(TableInputFormat.INPUT_TABLE, "tmp")var hBaseRDD = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat], classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable], classOf[org.apache.hadoop.hbase.client.Result])hBaseRDD.count()import scala.collection.JavaConverters._hBaseRDD.map(tuple => tuple._2).map(result => result.getColumn("cf".getBytes(), "val".getBytes())).map(keyValues => {( keyValues.asScala.reduceLeft {    (a, b) => if (a.getTimestamp > b.getTimestamp) a else b  }.getRow,  keyValues.asScala.reduceLeft {    (a, b) => if (a.getTimestamp > b.getTimestamp) a else b  }.getValue)}).take(10)hBaseRDD.map(tuple => tuple._2).map(result => (result.getRow, result.getColumn("cf".getBytes(), "val".getBytes()))).map(row => {(  row._1.map(_.toChar).mkString,  row._2.asScala.reduceLeft {    (a, b) => if (a.getTimestamp > b.getTimestamp) a else b  }.getValue.map(_.toChar).mkString)}).take(10)conf.set(TableInputFormat.INPUT_TABLE, "test1")//var hBaseRDD = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat], classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable], classOf[org.apache.hadoop.hbase.client.Result])hBaseRDD.map(tuple => tuple._2).map(result => (result.getRow, result.getColumn("lf".getBytes(), "app1".getBytes()))).map(row => if (row._2.size > 0) {(  row._1.map(_.toChar).mkString,  row._2.asScala.reduceLeft {    (a, b) => if (a.getTimestamp > b.getTimestamp) a else b  }.getValue.map(_.toInt).mkString)}).take(10)import java.nio.ByteBufferhBaseRDD.map(tuple => tuple._2).map(result => (result.getRow, result.getColumn("lf".getBytes(), "app1".getBytes()))).map(row => if (row._2.size > 0) {(  row._1.map(_.toChar).mkString,  ByteBuffer.wrap(row._2.asScala.reduceLeft {    (a, b) => if (a.getTimestamp > b.getTimestamp) a else b  }.getValue).getLong)}).take(10)//conf.set(TableInputFormat.SCAN_COLUMN_FAMILY, "lf")conf.set(TableInputFormat.SCAN_COLUMNS, "lf:app1")//var hBaseRDD = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat], classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable], classOf[org.apache.hadoop.hbase.client.Result])import java.nio.ByteBufferhBaseRDD.map(tuple => tuple._2).map(result => {  ( result.getRow.map(_.toChar).mkString,    ByteBuffer.wrap(result.value).getLong  )}).take(10)val conf = HBaseConfiguration.create()conf.set(TableInputFormat.INPUT_TABLE, "test1")var hBaseRDD = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat], classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable], classOf[org.apache.hadoop.hbase.client.Result])var rows = hBaseRDD.map(tuple => tuple._2).map(result => result.getRow.map(_.toChar).mkString)rows.map(row => row.split("\\|")).map(r => if (r.length > 1) (r(0), r(1)) else (r(0), "") ).groupByKey.take(10)