spark学习-SparkSQL--13-java版JavaRDD与JavaPairRDD的互相转换

1。引发：做一个java读取hbase的注册成表的程序。但是读出来的是javaPairRDD，而网上都是javaRDD转成dataFrame,我只能自己摸索怎么转成javaRDD
2。方法
　　JavaRDD => JavaPairRDD: 通过mapToPair函数
　　JavaPairRDD => JavaRDD: 通过map函数转换
3。不管其他先运行一下

package com.lcc.spark.rdd.test;import java.util.ArrayList;import java.util.Arrays;import java.util.List;import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaPairRDD;import org.apache.spark.api.java.JavaRDD;import org.apache.spark.api.java.JavaSparkContext;import org.apache.spark.api.java.function.Function;import org.apache.spark.api.java.function.Function2;import org.apache.spark.api.java.function.PairFunction;import org.apache.spark.api.java.function.VoidFunction;import org.apache.spark.storage.StorageLevel;import com.google.common.base.Optional;import scala.Tuple2;import shapeless.Tuple;  /**  * Hello world!  *  */  public class App {      public static void main(String[] args) {          SparkConf conf = new SparkConf().setMaster("local").setAppName("Simple Application");          JavaSparkContext sc = new JavaSparkContext(conf);          // convert from other RDD          JavaRDD<String> line1 = sc.parallelize(Arrays.asList("1 aa", "2 bb", "4 cc", "3 dd"));         line1.foreach(new VoidFunction<String>(){            @Override            public void call(String num) throws Exception {                // TODO Auto-generated method stub                System.out.println("numbers;"+num);            }        });        JavaPairRDD<String, String> prdd = line1.mapToPair(new PairFunction<String, String, String>() {              public Tuple2<String, String> call(String x) throws Exception {                  return new Tuple2(x.split(" ")[0], x.split(" ")[1]);              }          });          System.out.println("111111111111mapToPair:");          prdd.foreach(new VoidFunction<Tuple2<String, String>>() {              public void call(Tuple2<String, String> x) throws Exception {                  System.out.println(x);              }          });         /* JavaRDD => JavaPairRDD: 通过mapToPair函数        JavaPairRDD => JavaRDD: 通过map函数转换*/        System.out.println("===============1=========");          JavaRDD<String> javaprdd =prdd.map(new Function<Tuple2<String,String>,String>() {            private static final long serialVersionUID = 1L;            @Override            public String call(Tuple2<String, String> arg0)  {            // TODO Auto-generated method stub                System.out.println("arg0======================"+arg0);                System.out.println("arg0======================"+arg0._1);                 return arg0._1+" "+arg0._2;            }       });        System.out.println("===============2=========");          javaprdd.foreach(new VoidFunction<String>(){            @Override            public void call(String num) throws Exception {                // TODO Auto-generated method stub                System.out.println("numbers;"+num);            }        });    }  }  

结果如下

numbers;1 aanumbers;2 bbnumbers;4 ccnumbers;3 dd111111111111mapToPair:(1,aa)(2,bb)(4,cc)(3,dd)===============1========================2=========arg0======================(1,aa)arg0======================1numbers;1 aaarg0======================(2,bb)arg0======================2numbers;2 bbarg0======================(4,cc)arg0======================4numbers;4 ccarg0======================(3,dd)arg0======================3numbers;3 dd

4。分析

JavaPairRDD<String, String> prdd = line1.mapToPair(new PairFunction<要返回JavaPairRDD的key类型, 要返回JavaPairRDD的value类型, javaRDD要输入的类型>() {              public Tuple2<要返回JavaPairRDD的key类型, 要返回JavaPairRDD的value类型> call(javaRDD要输入的类型 x) throws Exception {                  return new Tuple2(x.split(" ")[0], x.split(" ")[1]);              }          });  JavaPairRDD<String, String> prdd = line1.mapToPair(new PairFunction<String, String, String>() {              public Tuple2<String, String> call(String x) throws Exception {                  return new Tuple2(x.split(" ")[0], x.split(" ")[1]);              }          });  

 JavaRDD<要返回JavaRDD的类型> javaprdd =prdd.map(new Function<Tuple2<JavaPairRDD的key类型,JavaPairRDD的value类型>,要返回JavaRDD的类型>() {            private static final long serialVersionUID = 1L;            @Override            public String call(Tuple2<String, String> arg0)  {            // TODO Auto-generated method stub                System.out.println("arg0======================"+arg0);                System.out.println("arg0======================"+arg0._1);                 return arg0._1+" "+arg0._2;            }       });        System.out.println("===============1=========");   JavaRDD<String> javaprdd =prdd.map(new Function<Tuple2<String,String>,String>() {            private static final long serialVersionUID = 1L;            @Override            public String call(Tuple2<String, String> arg0)  {            // TODO Auto-generated method stub                System.out.println("arg0======================"+arg0);                System.out.println("arg0======================"+arg0._1);                 return arg0._1+" "+arg0._2;            }       });        System.out.println("===============2=========");  

从输出结果看

===============1========================2=========比下面的map函数体的内容先输出，我在这里坑了很久，明明map函数写对了，但是我却没在下面打印输出，结果总是map函数体的没有执行，我就很纳闷啊，为什么啊，为什么不对啊，也没报错啊，但是我一打印才发现里面竟然执行了，这时候我突然想起spark的程序都是懒加载的，你用的时候他才会执行，所以上面两行会先打印出来。都是泪啊arg0======================(1,aa)arg0======================1numbers;1 aa

5。由此总结javaRDD怎么转换成JavaPairRDD的就，反着怎么转换回去，互为逆运算