spark 实战笔记case2

val t1 = sc.textFile("/tmp/db_case1/order_created/*").map(line => line.split("\t"))val t2 = sc.textFile("/tmp/db_case1/order_picked/*").map(line => line.split("\t"))val t3 = sc.textFile("/tmp/db_case1/order_shipped/*").map(line => line.split("\t"))val t1kv = t1.map(line => ( line(0), ( "created", line(1).substring(0, 19) ) ))val t2kv = t2.map(line => ( line(0), ( "picked" , line(1).substring(0, 19) ) ))val t3kv = t3.map(line => ( line(0), ( "shipped", line(1).substring(0, 19) ) ))def flatValues ( events:((String, (((String, String), Option[(String, String)]), Option[(String, String)]))) ): (String, String, String, String) = {  return ( events._1, events._2._1._1._2, events._2._1._2.getOrElse(("",""))._2, events._2._2.getOrElse(("",""))._2 )}t1kv.leftOuterJoin(t2kv).leftOuterJoin(t3kv).map(flatValues).take(10).foreach(println)def flatValues ( events:((String, (((String, String), Option[(String, String)]), Option[(String, String)]))) ): (String, (String, String), (String, String), (String, String)) = {  return ( events._1, events._2._1._1, events._2._1._2.getOrElse(("","")), events._2._2.getOrElse(("","")) )}t1kv.leftOuterJoin(t2kv).leftOuterJoin(t3kv).map(flatValues).take(10).foreach(println)def flatValues ( events:(String, ((String, String), Option[(String, String)])) ): (String, Seq[(String,String)]) = {  return ( events._1, Seq(events._2._1, events._2._2.getOrElse(("","")) ) )}t1kv.leftOuterJoin(t2kv).map(flatValues).take(10).foreach(println)def flatList ( events:(String, (Seq[(String, String)], Option[(String, String)])) ): (String, Seq[(String,String)]) = {  if (events._2._2.isEmpty)    return ( events._1, events._2._1 )  else    return ( events._1, events._2._1 :+ events._2._2.get )}t1kv.leftOuterJoin(t2kv).map(flatValues).leftOuterJoin(t3kv).map(flatList).take(10).foreach(println)def filterSLA ( events:Seq[(String,String)] ): Boolean = {  val eventMap = events.toMap  if ( (eventMap contains "created") && (eventMap contains "picked") ) {    val format = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss")    val ts1 = format.parse(eventMap("created")).getTime    val ts2 = format.parse(eventMap("picked")).getTime    if (ts2 - ts1 < 7200000)      false    else      true  } else {    true  }}t1kv.leftOuterJoin(t2kv).map(flatValues).leftOuterJoin(t3kv).map(flatList).filter( kv => filterSLA(kv._2) ).collect.foreach(println)t1kv.map(kv => (kv._1, Seq(kv._2))).leftOuterJoin(t2kv).map(flatList).leftOuterJoin(t3kv).map(flatList).filter( kv => filterSLA(kv._2) ).collect.foreach(println)t1kv.map(kv => (kv._1, Seq(kv._2))).leftOuterJoin(  t2kv.groupByKey.map(kv => (kv._1, (kv._2.toSeq.sortBy(_._2).toMap.keys.head, kv._2.toSeq.sortBy(_._2).toMap.values.head)))).map(flatList).leftOuterJoin(  t3kv.groupByKey.map(kv => (kv._1, (kv._2.toSeq.sortBy(_._2).toMap.keys.head, kv._2.toSeq.sortBy(_._2).toMap.values.head)))).map(flatList).collect.foreach(println)def flatGroup ( groups:(String, (Iterable[(String, String)])) ): (String, (String,String)) = {  val groupsMap = groups._2.toSeq.sortBy(_._2).toMap  return ( groups._1,  (groupsMap.keys.head, groupsMap.values.head) )}t1kv.map(kv => (kv._1, Seq(kv._2))).leftOuterJoin(  t2kv.groupByKey.map(flatGroup)).map(flatList).leftOuterJoin(  t3kv.groupByKey.map(flatGroup)).map(flatList).collect.foreach(println)t1kv.map(kv => (kv._1, Seq(kv._2))).leftOuterJoin(  t2kv.groupByKey.map(flatGroup)).map(flatList).leftOuterJoin(  t3kv.groupByKey.map(flatGroup)).map(flatList).filter( kv => filterSLA(kv._2) ).collect.foreach(println)