Spark:Scala实现action操作

来源:互联网 发布:阿里云备案管理系统 编辑:程序博客网 时间:2024/06/06 00:44

代码如下

package cn.spark.study.coreimport org.apache.spark.SparkConfimport org.apache.spark.SparkContextobject ActionOperation {  def main(args: Array[String]){    //reduce()    //collect()    //count()    take()    //saveAsTextFile()    //countByKey()    //foreach  }  def reduce(){    val conf =new SparkConf().setAppName("reduce").setMaster("local");val sc = new SparkContext(conf);val numberArray = Array(1,2,3,4,5,6,7,8,9,10);val numbers = sc.parallelize(numberArray, 1);val sum = numbers.reduce(_+_);println(sum)  }  def collect(){     val conf =new SparkConf().setAppName("reduce").setMaster("local");val sc = new SparkContext(conf);val numberArray = Array(1,2,3,4,5,6,7,8,9,10);val numbers = sc.parallelize(numberArray, 1);val doubleNumbers = numbers.map(num =>num *2)val doubleNumberArray = doubleNumbers.collect()for(num<-doubleNumberArray){  println(num)}  }  def count(){    val conf =new SparkConf().setAppName("reduce").setMaster("local");val sc = new SparkContext(conf);val numberArray = Array(1,2,3,4,5,6,7,8,9,10);val numbers = sc.parallelize(numberArray, 1);val count = numbers.count();println(count)  }def take(){val conf =new SparkConf().setAppName("reduce").setMaster("local");val sc = new SparkContext(conf);val numberArray = Array(1,2,3,4,5,6,7,8,9,10);val numbers = sc.parallelize(numberArray, 1);val top3number = numbers.take(3);println(top3number)//    for(num<- top3number){//      println(num)//    }  }   def saveAsTextFile(){    val conf =new SparkConf().setAppName("reduce").setMaster("local");val sc = new SparkContext(conf);val numberArray = Array(1,2,3,4,5,6,7,8,9,10);val numbers = sc.parallelize(numberArray, 1);val doubleNumbers = numbers.map(num =>num *2)doubleNumbers.saveAsTextFile("E://BigData//sparkdata//doule_number_scala")   }   def countByKey(){       val conf = new SparkConf().setAppName("countByKey").setMaster("local"); val sc = new SparkContext(conf); val studentList = Array(Tuple2("class1","leo"),Tuple2("class2","jack"),Tuple2("class1","tom"),Tuple2("class2","mike")) val students = sc.parallelize(studentList, 1) val studentCounts = students.countByKey(); println(studentCounts)   }}
原创粉丝点击