Spark算子（四）

Point 1：public class MapPartitonsWithIndexOperator {

package com.spark.operator;import java.util.ArrayList;import java.util.Arrays;import java.util.Iterator;import java.util.List;import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaRDD;import org.apache.spark.api.java.JavaSparkContext;import org.apache.spark.api.java.function.Function2;import org.apache.spark.api.java.function.VoidFunction;public class MapPartitonsWithIndexOperator {    public static void main(String[] args) {        SparkConf conf = new SparkConf().setAppName("MapPartitonsWithIndexOperator").setMaster(                "local[2]");        JavaSparkContext sc = new JavaSparkContext(conf);        // 准备一下数据        List<String> names = Arrays                .asList("xurunyun", "liangyongqi", "wangfei");        JavaRDD<String> nameRDD = sc.parallelize(names,2);        // 其实老师这个地方不写并行度2，默认其实它也是2         // parallelize并行集合的时候，指定了并行度为2，说白了就是numPartitions是2        // 也就是说我们上面的三大女神会被分到不同的两个分区里面去！        // 但是怎么分，我不知道，spark决定！！        // 如果我想知道谁和谁分到了一组里面去？        // MapPartitonsWithIndex这个算子可以拿到每个partition的index        JavaRDD<String> nameWithPartitonIndex = nameRDD.mapPartitionsWithIndex(new Function2<Integer, Iterator<String>, Iterator<String>>() {            private static final long serialVersionUID = 1L;            @Override            public Iterator<String> call(Integer index, Iterator<String> iterator)                    throws Exception {                List<String> list = new ArrayList<String>();                while(iterator.hasNext()){                    String name = iterator.next();                    String result = index + " : " + name;                    list.add(result);                }                return list.iterator();            }        }, true);        nameWithPartitonIndex.foreach(new VoidFunction<String>() {            private static final long serialVersionUID = 1L;            @Override            public void call(String result) throws Exception {                System.out.println(result);            }        });        sc.close();    }}

Point 2：MapPartitionsOperator

package com.spark.operator;import java.util.ArrayList;import java.util.Arrays;import java.util.HashMap;import java.util.Iterator;import java.util.List;import java.util.Map;import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaRDD;import org.apache.spark.api.java.JavaSparkContext;import org.apache.spark.api.java.function.FlatMapFunction;import org.apache.spark.api.java.function.VoidFunction;// 理解里面final使用的原因！public class MapPartitionsOperator {    public static void main(String[] args) {        SparkConf conf = new SparkConf().setAppName("JoinOperator")                .setMaster("local");        JavaSparkContext sc = new JavaSparkContext(conf);        // 准备一下数据        List<String> names = Arrays.asList("xurunyun","liangyongqi","wangfei");        JavaRDD<String> nameRDD = sc.parallelize(names);        final Map<String, Integer> scoreMap = new HashMap<String, Integer>();        scoreMap.put("xurunyun", 150);        scoreMap.put("liangyongqi", 100);        scoreMap.put("wangfei", 90);        // mapPartitions        // map算子，一次就处理一个partition的一条数据！！！        // mapPartitions算子，一次处理一个partition中所有的数据！！！        // 推荐的使用场景！！！        // 如果你的RDD的数据不是特别多，那么采用MapPartitions算子代替map算子，可以加快处理速度        // 比如说100亿条数据，你一个partition里面就有10亿条数据，不建议使用mapPartitions，        // 内存溢出        JavaRDD<Integer> scoreRDD = nameRDD.mapPartitions(new FlatMapFunction<Iterator<String>, Integer>() {            private static final long serialVersionUID = 1L;            @Override            public Iterable<Integer> call(Iterator<String> iterator)                    throws Exception {                List<Integer> list = new ArrayList<Integer>();                while(iterator.hasNext()){                    String name = iterator.next();                    Integer score = scoreMap.get(name);                    list.add(score);                }                return list;            }        });        scoreRDD.foreach(new VoidFunction<Integer>() {            private static final long serialVersionUID = 1L;            @Override            public void call(Integer score) throws Exception {                System.out.println(score);            }        });        sc.close();    }}

Point 3：MapOperator

package com.spark.operator;import java.util.Arrays;import java.util.List;import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaRDD;import org.apache.spark.api.java.JavaSparkContext;import org.apache.spark.api.java.function.Function;import org.apache.spark.api.java.function.VoidFunction;public class MapOperator {    public static void main(String[] args) {        SparkConf conf = new SparkConf().setAppName("LineCount")                .setMaster("local");        JavaSparkContext sc = new JavaSparkContext(conf);        List<Integer> numbers = Arrays.asList(1,2,3,4,5);        JavaRDD<Integer> numberRDD = sc.parallelize(numbers);        // map对每个元素进行操作        JavaRDD<Integer> results = numberRDD.map(new Function<Integer, Integer>() {            private static final long serialVersionUID = 1L;            @Override            public Integer call(Integer number) throws Exception {                return number * 10;            }        });        results.foreach(new VoidFunction<Integer>() {            private static final long serialVersionUID = 1L;            @Override            public void call(Integer result) throws Exception {                System.out.println(result);            }        });        sc.close();    }}