spark--transform算子--mapPartitionsWithIndex

import org.apache.spark.{SparkConf, SparkContext}import scala.collection.mutable.ArrayBuffer/**  * Created by liupeng on 2017/6/15.  */object T_mapPartitionsWithIndex {  System.setProperty("hadoop.home.dir","F:\\hadoop-2.6.5")  def fun_index(index : Int, iter : Iterator[String]) : Iterator[String] = {    var list  = ArrayBuffer[String]()    while (iter.hasNext)    {      val name : String = iter.next()      var fs = index + ":" + name      list += fs      println(fs)    }    return list.iterator  }  def main(args: Array[String]): Unit = {    val conf = new SparkConf().setAppName("mapPartitionsWithIndex_test").setMaster("local")    val sc = new SparkContext(conf)    //准备一下数据    val names: List[String] = List("liupeng", "xuliuxi", "xiaoma")    val nameRDD = sc.parallelize(names, 2)    //  按照分区以及索引遍历    //如果想知道谁分到了一起，mapPartitionsWithIndex这个算子可以拿到每个partition的index    val nameWithPartionIndex = nameRDD.mapPartitionsWithIndex(fun_index)    println(nameWithPartionIndex.count())  }}

运行结果：

0:liupeng

1:xuliuxi
1:xiaoma

3