Storm1.1.0<温故而知新--hdfs和storm的集成>

1.环境

apache-storm-1.1.0Hadoop 2.8.0

使用到的依赖：

<dependencies>        <dependency>            <groupId>org.apache.storm</groupId>            <artifactId>storm-core</artifactId>            <version>1.1.0</version>        </dependency>        <dependency>            <groupId>org.apache.storm</groupId>            <artifactId>storm-hdfs</artifactId>            <version>1.1.0</version>        </dependency>    </dependencies>

需求：读取hdfs上的mapreduce日志文件，统计其中的INFO、WARN、DEBUG、Error日志级别的条数

思路很简单，Spout端读入文件，然后Bolt端做Wordcount（先用正则表达式匹配日志级别）

2.实现

LogLevelCountTopology

用到的参数：

import neu.bolt.CountBolt;import neu.bolt.ExtractBolt;import org.apache.storm.Config;import org.apache.storm.LocalCluster;import org.apache.storm.generated.AlreadyAliveException;import org.apache.storm.generated.AuthorizationException;import org.apache.storm.generated.InvalidTopologyException;import org.apache.storm.hdfs.spout.HdfsSpout;import org.apache.storm.hdfs.spout.TextFileReader;import org.apache.storm.topology.TopologyBuilder;import org.apache.storm.tuple.Fields;import java.util.HashMap;public class LogLevelCountTopology {    public static void main(String[] args) throws InvalidTopologyException, AuthorizationException, AlreadyAliveException, InterruptedException {        System.setProperty("HADOOP_USER_NAME", "root");        if (args.length != 4) {            System.out.println("Usage <HdfsUri SourceDir ArchiveDir BadFilesDir>");            System.exit(1);        }        TopologyBuilder builder = new TopologyBuilder();        HdfsSpout hdfsSpout = new HdfsSpout()                .setReaderType("text")                .withOutputFields(TextFileReader.defaultFields)                .setHdfsUri(args[0])                .setSourceDir(args[1])                .setArchiveDir(args[2])                .setBadFilesDir(args[3]);        HashMap<String, Object> hashMap = new HashMap<>();        //正则表达式        hashMap.put(ExtractBolt.REGEX, ".{23}(INFO|DEBUG|WARN|ERROR)");        hashMap.put(ExtractBolt.FIELD, "line");        builder.setSpout("hdfsSpout", hdfsSpout, 1);        builder.setBolt("extractbolt", new ExtractBolt(), 1)                .addConfigurations(hashMap).shuffleGrouping("hdfsSpout");        builder.setBolt("countBolt", new CountBolt(), 1)                .fieldsGrouping("extractbolt", new Fields("level"));        Config conf = new Config();        conf.setDebug(true);        conf.setMaxTaskParallelism(1);        LocalCluster cluster = new LocalCluster();        cluster.submitTopology("hdfsLogLevelCountTopology", conf, builder.createTopology());        Thread.sleep(90000);        cluster.shutdown();    }}

ExtractBolt

import org.apache.storm.task.OutputCollector;import org.apache.storm.task.TopologyContext;import org.apache.storm.topology.IRichBolt;import org.apache.storm.topology.OutputFieldsDeclarer;import org.apache.storm.tuple.Fields;import org.apache.storm.tuple.Tuple;import org.apache.storm.tuple.Values;import java.util.Map;import java.util.regex.Matcher;import java.util.regex.Pattern;public class ExtractBolt implements IRichBolt {    public static final String REGEX = "regex";    public static final String FIELD = "field";    String field;    Pattern regex;    OutputCollector collector;    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {        String regexString = (String) stormConf.get(REGEX);        this.collector = collector;        this.field = (String) stormConf.get(FIELD);        this.regex = Pattern.compile(regexString);    }    public void execute(Tuple input) {        String log = input.getStringByField(field);        if (log != null) {            Matcher matcher = regex.matcher(log);            if (matcher.find()) {                String level = matcher.group(1);                collector.emit(new Values(level));            } else {                System.err.println("不包含INFO|DEBUG|ERROR|WARN 日志：" + log);            }        }        collector.ack(input);    }    public void cleanup() {    }    public void declareOutputFields(OutputFieldsDeclarer declarer) {        declarer.declare(new Fields("level"));    }    public Map<String, Object> getComponentConfiguration() {        return null;    }}

CountBolt

import org.apache.storm.topology.BasicOutputCollector;import org.apache.storm.topology.OutputFieldsDeclarer;import org.apache.storm.topology.base.BaseBasicBolt;import org.apache.storm.tuple.Fields;import org.apache.storm.tuple.Tuple;import org.apache.storm.tuple.Values;import java.util.HashMap;import java.util.Map;public class CountBolt extends BaseBasicBolt {    private Map<String, Integer> counts = new HashMap<>();    public void execute(Tuple input, BasicOutputCollector collector) {        String level = input.getStringByField("level");        Integer count = counts.get(level);        if (count == null)            count = 0;        count++;        counts.put(level, count);        System.out.println(level + " :  " + count);        collector.emit(new Values(level, count));    }    public void declareOutputFields(OutputFieldsDeclarer declarer) {        declarer.declare(new Fields("level", "count"));    }}

主函数传参示例hdfs://172.17.11.85:9000 /log /ArchiveDir /BadFilesDir

给一张IDE控制台的输出日志：