STORM入门之（集成KafkaBolt）

来源：互联网发布：网络创世纪手游编辑：程序博客网时间：2024/06/08 17:24

根据第一篇文章：STORM入门之（Topology简易Demo）进行扩展集成Kafka

STORM入门之（Topology简易Demo）传送门：http://blog.csdn.net/yl3395017/article/details/77449275

新增Bolt：

package com.storm.bolt;import java.util.Map;import org.apache.storm.task.OutputCollector;import org.apache.storm.task.TopologyContext;import org.apache.storm.topology.OutputFieldsDeclarer;import org.apache.storm.topology.base.BaseRichBolt;import org.apache.storm.tuple.Fields;import org.apache.storm.tuple.Tuple;import org.apache.storm.tuple.Values;public class KafkaMsgBolt extends BaseRichBolt{    private OutputCollector collector;    @Override    public void execute(Tuple arg0) {        String message = (String) arg0.getValue(0);        System.out.print("kafka message is:"+message);        collector.emit(new Values("forward",message));     }    @Override    public void prepare(Map arg0, TopologyContext arg1, OutputCollector arg2) {        collector = arg2;      }    @Override    public void declareOutputFields(OutputFieldsDeclarer arg0) {        arg0.declare(new Fields("key", "message"));            }    }

Topology新增静态方法，用于配置KafkaBolt

  /**     * 构建KafkaBolt     * metadata.broker.list=10.2.4.13:9092,10.2.4.14:9092,10.2.4.12:9092     * bootstrap.servers=10.2.4.13:9092,10.2.4.14:9092,10.2.4.12:9092     * producer.type=async     * request.required.ack=1     * serializer.class=kafka.serializer.StringEncoder     * key.serializer=org.apache.kafka.common.serialization.StringSerializer     * value.serializer=org.apache.kafka.common.serialization.StringSerializer     * sendtopic=test     */    private static void builtKafkaBolt(TopologyBuilder builder){        //kafka producer config        Properties prop = new Properties();        prop.put("metadata.broker.list", "10.2.4.13:9092,10.2.4.14:9092,10.2.4.12:9092");        prop.put("bootstrap.servers", "10.2.4.13:9092,10.2.4.14:9092,10.2.4.12:9092");        prop.put("producer.type","async");        prop.put("request.required.acks","1");        prop.put("serializer.class", "kafka.serializer.StringEncoder");        prop.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");        prop.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");        //kafkaBolt        KafkaBolt bolt = new KafkaBolt();        bolt.withTopicSelector(new DefaultTopicSelector("test_rce_yjd"));        bolt.withProducerProperties(prop);        bolt.withTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper());        //构建KafkaBolt        builder.setBolt("msgSentenceBolt", new KafkaMsgBolt()).shuffleGrouping("BoltA");        //转发kafka消息        builder.setBolt("forwardToKafka", bolt).shuffleGrouping("msgSentenceBolt");    }

修正Topology

public class Topology {    private static TopologyBuilder builder = null;    public static void main(String args[]) throws AuthorizationException, AlreadyAliveException, InvalidTopologyException {        builder = new TopologyBuilder();        builder.setSpout("SpoutA",new SpoutA(),1);        builder.setBolt("BoltA",new BoltA(),1).shuffleGrouping("SpoutA");        //构建KafkaBolt        builtKafkaBolt(builder);        Config conf = new Config();        conf.setDebug(true);        if (args != null && args.length > 0) {            conf.setNumWorkers(3);            StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());        }        else {            LocalCluster cluster = new LocalCluster();            cluster.submitTopology("soc", conf, builder.createTopology());        }    }

结果查看

KAFKA查看命令（根据实际情况自己修正路径）：

/kafka/bin/kafka-console-consumer.sh --zookeeper 10.2.4.12:2181,10.2.4.13:2181,10.2.4.14:2181 --topic test_rce_yjd

阅读全文

1 0