Flink实现WordCount

来源:互联网 发布:java写入txt文件 编辑:程序博客网 时间:2024/06/08 19:38
import org.apache.flink.api.common.functions.FlatMapFunction;import org.apache.flink.api.java.DataSet;import org.apache.flink.api.java.ExecutionEnvironment;import org.apache.flink.api.java.tuple.Tuple2;import org.apache.flink.util.Collector;public class WordCount {    public static void main(String[] args) throws Exception {        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();        DataSet<String> text = env.fromElements("To be, or not to be,--that is the question:--",                                                "Whether 'tis nobler in the mind to suffer",                                                "The slings and arrows of outrageous fortune",                                                "Or to take arms against a sea of troubles,");        DataSet<Tuple2<String, Integer>> counts = text.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {            @Override            public void flatMap(String s, Collector<Tuple2<String, Integer>> collector) throws Exception {                String[] tokens = s.toLowerCase().split("\\s+");                for (String token : tokens) {                    if (token.length() > 0) {                        collector.collect(new Tuple2<String, Integer>(token, 1));                    }                }            }        }).groupBy(0).sum(1);        counts.print();    }}
0 0