<spark>新手helloworld

来源：互联网发布：战地2网络对战平台编辑：程序博客网时间：2024/05/01 17:42
world count 程序
/** * Created by hadoop on 17-2-23. */import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaPairRDD;import org.apache.spark.api.java.JavaRDD;import org.apache.spark.api.java.JavaSparkContext;import org.apache.spark.api.java.function.FlatMapFunction;import org.apache.spark.api.java.function.Function2;import org.apache.spark.api.java.function.PairFunction;import scala.Tuple2;import java.util.Arrays;public class JavaWordCount {    public static void main(String[] args)throws Exception {        SparkConf conf = new SparkConf().setAppName("wordCount");        JavaSparkContext sc = new JavaSparkContext(conf);        JavaRDD<String> input = sc.textFile("/home/hadoop/input/idea");        JavaRDD<String> words = input.flatMap(                new FlatMapFunction<String, String>() {                    @Override                    public Iterable<String> call(String s) throws Exception {                        return Arrays.asList(s.split(" "));                    }                }        );        JavaPairRDD<String,Integer> counts = words.mapToPair(                new PairFunction<String, String, Integer>() {                    @Override                    public Tuple2<String, Integer> call(String s) throws Exception {                        return new Tuple2(s,1);                    }                }        ).reduceByKey(                new Function2<Integer, Integer, Integer>() {                    @Override                    public Integer call(Integer integer, Integer integer2) throws Exception {                        return integer+integer2;                    }                }        );        counts.saveAsTextFile("/home/hadoop/output");    }}
0 0