spark定制之二：start.scala中的DSL

来源：互联网发布：幼儿教师网络研修总结编辑：程序博客网时间：2024/05/29 03:09

通过类的隐式转换可以让写脚本变得更容易（写入"select * from testperson" hqlgo直接处结果，"select * from testperson" saveto somefile直接写入文件）：

def getRegisterString(rddname:String,classname:String,tablename:String,tabledef:String) : String = {    val members = tabledef.split(",").map(_.trim.split(" ").filter(""!=)).map(x => (x(0).trim,x(1).trim.head.toString.toUpperCase+x(1).trim.tail))    val classmemberdef = members.map(x => (x._1+":"+x._2)).mkString(",")    val convertstr = members.map(x => x._2).zipWithIndex.map(x => "t("+x._2+").to"+x._1).mkString(",")    return s"""        case class ${classname}(${classmemberdef})        val schemardd = ${rddname}.map(_.split("${FIELD_SEPERATOR}")).map(t=>${classname}(${convertstr}))        hive.registerRDDAsTable(schemardd,"${tablename}")    """}org.apache.spark.repl.Main.interp.command("""class CommandTranslator(cmd:String) extends java.io.Serializable {    def hqlgo()(implicit f: SchemaRDD => MySchemaRDD) = {        lastrdd = hql(cmd)        lastrdd.go()    }    def hqlsaveto(output: String)(implicit f: SchemaRDD => MySchemaRDD) = {        lastrdd = hql(cmd)        lastrdd.saveto(output)    }    def defineas(tabledef:String) = {        if( tabledef != "" ) {            org.apache.spark.repl.Main.interp.command(                getRegisterString(cmd,cmd.toUpperCase,cmd,tabledef)            )        } else {            org.apache.spark.repl.Main.interp.command(                "hive.registerRDDAsTable(${cmd},\"${cmd}\")"            )        }    }    def from(filepath:String) {        if( cmd.startsWith("create table ") ) {            val tablename = cmd.substring(13).trim().split(" ")(0)            val leftstr = cmd.substring(13).trim().substring(tablename.length).trim()            val tabledef = leftstr.substring(1,leftstr.length-1).trim()            val realfile = AutoFileUtil.regularFile(filepath)            org.apache.spark.repl.Main.interp.command(                "val "+tablename+" = sc.textFile(\""+realfile+"\")"            )            new CommandTranslator(tablename).defineas(tabledef)        } else {            println("usage:")            println("\"create table sometablename (field1 string,field2 int...)\" from \"somefile or hdfs:somepath\"")        }    }}object CommandTranslator {    implicit def stringToTranslator(cmd:String) = new CommandTranslator(cmd)    def show(tabledata:Array[org.apache.spark.sql.Row]) = {        tabledata.foreach( x => println(x.mkString("\t")))    }}""")def auto = CommandTranslatorimport CommandTranslator._def help = {    println("""example:        "select * from testperson" hqlgo        "select * from testperson" hqlsaveto "hdfs://somedir"        "select * from testperson" hqlsaveto "somelocalfile"        "create table sometable (name string,age int,weight double)" from "hdfs:/test/testperson"        auto show hqlresult        "somerdddata" defineas "(name string,age int)"        if you want to see the help of enveronment, please type :help        """)}

0 0