Storm Trident+DRPC实例

需求

Trident实现用户使用浏览器统计
通过DRPC查看结果

开发过程

实现IBatchSpout批量读取日志文件

public class MyBatchSpout implements IBatchSpout {    Fields fields;    HashMap<Long, List<List<Object>>> batches = new HashMap();    public MyBatchSpout(Fields fields) {        this.fields = fields;    }    @Override    public void open(Map map, TopologyContext topologyContext) {    }    @Override    public void emitBatch(long batchId, TridentCollector tridentCollector) {        List<List<Object>> batch = (List) this.batches.get(Long.valueOf(batchId));        if (null == batch) {            batch = new ArrayList<List<Object>>();            //读取日志文件列表            String dataDir = "logs/";            File file = new File(dataDir);            Collection<File> listFiles = FileUtils.listFiles(file, new String[]{"log"}, true);            for (File f : listFiles) {                List<String> readLines = null;                try {                    readLines = FileUtils.readLines(f);                    for (String line : readLines) {                        batch.add(new Values(line));                    }                } catch (IOException e) {                    e.printStackTrace();                }                // 文件已经处理完成,在末尾添加done和时间戳，避免重复读取                try {                    File srcFile = f.getAbsoluteFile();                    File destFile = new File(srcFile + ".done." + System.currentTimeMillis());                    FileUtils.moveFile(srcFile, destFile);                } catch (Exception e) {                    e.printStackTrace();                }                this.batches.put(batchId, batch);            }        }        for (List<Object> list : batch) {            tridentCollector.emit(list);        }    }    @Override    public void ack(long batchId) {        this.batches.remove(Long.valueOf(batchId));    }    @Override    public void close() {    }    @Override    public Map<String, Object> getComponentConfiguration() {        Config conf = new Config();        conf.setMaxTaskParallelism(1);        return conf;    }    @Override    public Fields getOutputFields() {        return this.fields;    }}

实现BaseFunction，从每行日志中分割出IP地址和浏览器信息，合并成一个新字段IP_browser

public static class Split extends BaseFunction{        @Override        public void execute(TridentTuple tridentTuple, TridentCollector tridentCollector) {            String line=tridentTuple.getString(0);            String[] words=line.split("\"");            String IP=words[0].split(" ")[0];            String browser=words[5];            if (!browser.equals("-")){                String IP_browser=IP+"_"+browser;                tridentCollector.emit(new Values(IP_browser));            }        }    }

实现BaseAggregator，剔除重复的IP_browser，发送新字段IP和数量1

public static class Aggregate extends BaseAggregator<Map<String,Integer>>{        @Override        public Map<String,Integer> init(Object o, TridentCollector tridentCollector) {            return new HashMap<String,Integer>() ;        }        @Override        public void aggregate(Map<String,Integer> map, TridentTuple tridentTuple, TridentCollector tridentCollector) {            String IP_browser=tridentTuple.getStringByField("IP_browser");            String[] words=IP_browser.split("_");           String IP=words[0];            map.put(IP,1);        }        @Override        public void complete(Map<String,Integer> map, TridentCollector tridentCollector) {            for (Map.Entry<String,Integer> e:map.entrySet()){                tridentCollector.emit(new Values(e.getKey()));            }        }    }

创建TridentTopology，提供用于构建Trident实时计算程序的一些接口
创建一个TridentState对象，通过newStream方法从指定的Spout创建一个新的数据输入流
日志拆分合并新字段->聚合去重复->count统计数量
创建DRPC流，函数命名为browserFunction

public static StormTopology buildTopology(LocalDRPC drpc) {        TridentTopology topology = new TridentTopology();        MyBatchSpout spout = new MyBatchSpout(new Fields("line"));        TridentState tridentState=topology.newStream("spout",spout)                .parallelismHint(4)                .each(new Fields("line"),new Split(),new Fields("IP_browser"))                .parallelismHint(4)                .groupBy(new Fields("IP_browser"))                .partitionAggregate(new Fields("IP_browser"),new Aggregate(),new Fields("IP"))                .toStream()                .parallelismHint(4)                .groupBy(new Fields("IP"))                .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"));        topology.newDRPCStream("browserFunction",drpc)                .stateQuery(tridentState,new Fields("args"),new MapGet(),new Fields("count"))                .each(new Fields("count"),new FilterNull());        return topology.build();    }

本地模式运行DRPC

Config conf = new Config();        conf.setMaxSpoutPending(20);        if (args.length == 0) {            LocalDRPC drpc = new LocalDRPC();            //创建LocalDRPC对象在进程内模拟一个DRPC服务器（类似于LocalCluster在进程内模拟一个Storm集群）            LocalCluster cluster = new LocalCluster();            //创建LocalCluster对象在本地模式运行topology            cluster.submitTopology("wordCounter", conf, buildTopology(drpc));            for (int i = 0; i < 100; i++) {                System.out.println("DRPC RESULT: " + drpc.execute("browserFunction", "114.112.141.6"));                Thread.sleep(1000);            }

运行结果