KAFKA知识（Consumer）

来源：互联网发布：大数据四大特征编辑：程序博客网时间：2024/06/05 15:59

KAFKA版本

kafka_2.11-0.10.0.1 代码原理主要是建立线程池，利用多线程来消费信息，原理还是比较简单的，可配置线程数

KafkaConsumer

import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Properties;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import kafka.consumer.Consumer;import kafka.consumer.ConsumerConfig;import kafka.consumer.KafkaStream;import kafka.javaapi.consumer.ConsumerConnector;public class KafkaConsumer {        private final ConsumerConnector consumer;    private final String topic;    private ExecutorService executor;        public KafkaConsumer(String a_zookeeper, String a_groupId, String a_topic) {        consumer = Consumer.createJavaConsumerConnector(createConsumerConfig(a_zookeeper,a_groupId));        this.topic = a_topic;    }    public void shutdown() {        if (consumer != null)            consumer.shutdown();        if (executor != null)            executor.shutdown();    }    public void run(int numThreads) {        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();        topicCountMap.put(topic, new Integer(numThreads));        Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer                .createMessageStreams(topicCountMap);        List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topic);        executor = Executors.newFixedThreadPool(numThreads);        int threadNumber = 0;        for (final KafkaStream stream : streams) {            executor.submit(new ConsumerMsgTask(stream, threadNumber));            threadNumber++;        }    }    private static ConsumerConfig createConsumerConfig(String a_zookeeper,String a_groupId) {        Properties props = new Properties();        props.put("zookeeper.connect", a_zookeeper);        props.put("group.id", a_groupId);        props.put("zookeeper.session.timeout.ms", "4000");        props.put("zookeeper.sync.time.ms", "200");        props.put("auto.commit.interval.ms", "1000");        return new ConsumerConfig(props);    }          public static void main(String[] arg) {        String[] args = { "10.2.4.12:2181,10.2.4.13:2181,10.2.4.14:2181", "logFramework", "test_rce_yjd", "15" };        String zooKeeper = args[0];        String groupId = args[1];        String topic = args[2];        int threads = Integer.parseInt(args[3]);        KafkaConsumer demo = new KafkaConsumer(zooKeeper, groupId, topic);        demo.run(threads);    }}

ConsumerMsgTask

import java.io.IOException;import java.util.ArrayList;import java.util.LinkedList;import java.util.List;import com.neusoft.im.ex.HBaseImportEx;import kafka.consumer.ConsumerIterator;import kafka.consumer.KafkaStream;public class ConsumerMsgTask implements Runnable {    private KafkaStream m_stream;    private int m_threadNumber;    public ConsumerMsgTask(KafkaStream stream, int threadNumber) {        m_threadNumber = threadNumber;        m_stream = stream;    }    public void run() {        LinkedList l = new LinkedList();        ConsumerIterator<byte[], byte[]> it = m_stream.iterator();        while (it.hasNext()){                      System.out.println("消费 " + m_threadNumber + ": "+ new String(it.next().message()));        }                }}

结果

测试联调

其实我们也可以根据以前Storm专题进行联调测试文章传送门：http://blog.csdn.net/yl3395017/article/details/77452604 利用KafkaBolt生产数据，Kafka消费监听结果所下图

阅读全文

1 0