STORM入门之（集成KafkaSpout）

此篇基于原有两篇文章基础上扩展

STORM入门之（集成KafkaBolt） 传送门：http://blog.csdn.net/yl3395017/article/details/77452604

KafkaSpout更新

主要是构建KafkaSpout基本配置操作

/**     * 构建KafkaSpout     */    private static void builtKafkaSpout(TopologyBuilder builder) {        try {            BrokerHosts brokerHosts = new ZkHosts("10.2.4.12:2181,10.2.4.13:2181,10.2.4.14:2181");            SpoutConfig spoutConf = new SpoutConfig(brokerHosts, "test_rce_yjd", "/storm/data/2016122615", "logFramework");            spoutConf.scheme = new SchemeAsMultiScheme(new StringScheme());            spoutConf.zkPort = 2181;            spoutConf.ignoreZkOffsets = false; // 每次消费都从头开始，用于性能测试            List<String> servers = new ArrayList<String>();            String ZK_SERVERS = "10.2.4.12,10.2.4.13,10.2.4.14";            if (ZK_SERVERS != null) {                String[] arr = ZK_SERVERS.split(",");                for (int i = 0; i < arr.length; i++) {                    if (!("".equals(arr[i]))) {                        servers.add(arr[i]);                    }                }            }            spoutConf.zkServers = servers;            builder.setSpout("kafka-spout", new KafkaSpout(spoutConf),1);        } catch (Exception e) {        }    }

Topology更新

构建Spout

package com.storm.topology;import com.storm.bolt.BoltA;import com.storm.bolt.KafkaMsgBolt;import com.storm.spout.SpoutA;import org.apache.storm.Config;import org.apache.storm.LocalCluster;import org.apache.storm.StormSubmitter;import org.apache.storm.generated.AlreadyAliveException;import org.apache.storm.generated.AuthorizationException;import org.apache.storm.generated.InvalidTopologyException;import org.apache.storm.kafka.*;import org.apache.storm.kafka.bolt.KafkaBolt;import org.apache.storm.kafka.bolt.mapper.FieldNameBasedTupleToKafkaMapper;import org.apache.storm.kafka.bolt.selector.DefaultTopicSelector;import org.apache.storm.spout.SchemeAsMultiScheme;import org.apache.storm.topology.TopologyBuilder;import org.apache.storm.utils.Utils;import java.util.ArrayList;import java.util.List;import java.util.Properties;/** * Created with IntelliJ IDEA. * User: Administrator * Date: 17-8-21 * Time: 上午10:54 * To change this template use File | Settings | File Templates. */public class Topology {    public static boolean  kafka = true;    public static void main(String args[]) throws AuthorizationException, AlreadyAliveException, InvalidTopologyException {        TopologyBuilder builder = new TopologyBuilder();        if(kafka){            //KafkaSpout            builtKafkaSpout(builder);            builder.setBolt("BoltA",new BoltA(),1).shuffleGrouping("kafka-spout");        }else{            //普通Spout            builder.setSpout("SpoutA",new SpoutA(),1);            builder.setBolt("BoltA",new BoltA(),1).shuffleGrouping("SpoutA");            //构建KafkaBolt            builtKafkaBolt(builder);        }        Config conf = new Config();        conf.setDebug(true);        if (args != null && args.length > 0) {            conf.setNumWorkers(3);            StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());        }        else {            LocalCluster cluster = new LocalCluster();            cluster.submitTopology("soc", conf, builder.createTopology());        }    }

整体代码

package com.storm.topology;import com.storm.bolt.BoltA;import com.storm.bolt.KafkaMsgBolt;import com.storm.spout.SpoutA;import org.apache.storm.Config;import org.apache.storm.LocalCluster;import org.apache.storm.StormSubmitter;import org.apache.storm.generated.AlreadyAliveException;import org.apache.storm.generated.AuthorizationException;import org.apache.storm.generated.InvalidTopologyException;import org.apache.storm.kafka.*;import org.apache.storm.kafka.bolt.KafkaBolt;import org.apache.storm.kafka.bolt.mapper.FieldNameBasedTupleToKafkaMapper;import org.apache.storm.kafka.bolt.selector.DefaultTopicSelector;import org.apache.storm.spout.SchemeAsMultiScheme;import org.apache.storm.topology.TopologyBuilder;import org.apache.storm.utils.Utils;import java.util.ArrayList;import java.util.List;import java.util.Properties;/** * Created with IntelliJ IDEA. * User: Administrator * Date: 17-8-21 * Time: 上午10:54 * To change this template use File | Settings | File Templates. */public class Topology {    public static boolean  kafka = true;    public static void main(String args[]) throws AuthorizationException, AlreadyAliveException, InvalidTopologyException {        TopologyBuilder builder = new TopologyBuilder();        if(kafka){            //KafkaSpout            builtKafkaSpout(builder);            builder.setBolt("BoltA",new BoltA(),1).shuffleGrouping("kafka-spout");        }else{            //普通Spout            builder.setSpout("SpoutA",new SpoutA(),1);            builder.setBolt("BoltA",new BoltA(),1).shuffleGrouping("SpoutA");            //构建KafkaBolt            builtKafkaBolt(builder);        }        Config conf = new Config();        conf.setDebug(true);        if (args != null && args.length > 0) {            conf.setNumWorkers(3);            StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());        }        else {            LocalCluster cluster = new LocalCluster();            cluster.submitTopology("soc", conf, builder.createTopology());//            Utils.sleep(20000);//            cluster.killTopology("soc");//            cluster.shutdown();        }    }    /**     * 构建KafkaBolt     * metadata.broker.list=10.2.4.13:9092,10.2.4.14:9092,10.2.4.12:9092     * bootstrap.servers=10.2.4.13:9092,10.2.4.14:9092,10.2.4.12:9092     * producer.type=async     * request.required.ack=1     * serializer.class=kafka.serializer.StringEncoder     * key.serializer=org.apache.kafka.common.serialization.StringSerializer     * value.serializer=org.apache.kafka.common.serialization.StringSerializer     * sendtopic=test     */    private static void builtKafkaBolt(TopologyBuilder builder){        //kafka producer config        Properties prop = new Properties();        prop.put("metadata.broker.list", "10.2.4.13:9092,10.2.4.14:9092,10.2.4.12:9092");        prop.put("bootstrap.servers", "10.2.4.13:9092,10.2.4.14:9092,10.2.4.12:9092");        prop.put("producer.type","async");        prop.put("request.required.acks","1");        prop.put("serializer.class", "kafka.serializer.StringEncoder");        prop.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");        prop.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");        //kafkaBolt        KafkaBolt bolt = new KafkaBolt();        bolt.withTopicSelector(new DefaultTopicSelector("test_rce_yjd"));        bolt.withProducerProperties(prop);        bolt.withTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper());        //构建KafkaBolt        builder.setBolt("msgSentenceBolt", new KafkaMsgBolt()).shuffleGrouping("BoltA");        //转发kafka消息        builder.setBolt("forwardToKafka", bolt).shuffleGrouping("msgSentenceBolt");    }    /**     * 构建KafkaSpout     */    private static void builtKafkaSpout(TopologyBuilder builder) {        try {            BrokerHosts brokerHosts = new ZkHosts("10.2.4.12:2181,10.2.4.13:2181,10.2.4.14:2181");            SpoutConfig spoutConf = new SpoutConfig(brokerHosts, "test_rce_yjd", "/storm/data/2016122615", "logFramework");            spoutConf.scheme = new SchemeAsMultiScheme(new StringScheme());            spoutConf.zkPort = 2181;            spoutConf.ignoreZkOffsets = false; // 每次消费都从头开始，用于性能测试            List<String> servers = new ArrayList<String>();            String ZK_SERVERS = "10.2.4.12,10.2.4.13,10.2.4.14";            if (ZK_SERVERS != null) {                String[] arr = ZK_SERVERS.split(",");                for (int i = 0; i < arr.length; i++) {                    if (!("".equals(arr[i]))) {                        servers.add(arr[i]);                    }                }            }            spoutConf.zkServers = servers;            builder.setSpout("kafka-spout", new KafkaSpout(spoutConf),1);        } catch (Exception e) {        }    }}

结果