kaka消费者

来源：互联网发布：隔墙听音器淘宝编辑：程序博客网时间：2024/05/17 03:54

引入Kafka依赖包

<dependencies>      <dependency>          <groupId>org.apache.kafka</groupId>          <artifactId>kafka_2.11</artifactId>          <version>0.8.2.1</version>      </dependency>      <dependency>          <groupId>org.apache.kafka</groupId>          <artifactId>kafka-clients</artifactId>          <version>0.8.2.1</version>      </dependency>  </dependencies>

kafka消费者配置

zookeeper.connect=192.168.27.129:2181zookeeper.session.timeout.ms=1000zookeeper.sync.time.ms=500auto.commit.interval.ms=500auto.commit.enable=falseauto.offset.reset=smallest#group.id=ecsgroupgroup.id=test-group-1#topic=ecps-test-05topic=test#topic=ecps-test2derializer.class=com.ai.iis.logger.kafka.MessageDeserialize#equals to kafka topic partitions valuethread=1#partitioner.class=

kafka消费者，该类是一个抽象类，其抽象方法为doBusiness为业务处理方法，其参数为kafka的每次消息

package com.ai.ecsite.kafka.consumer.interfaces;import com.ai.ecsite.util.common.PropertiesLoader;import kafka.consumer.ConsumerConfig;import kafka.consumer.ConsumerIterator;import kafka.consumer.KafkaStream;import kafka.javaapi.consumer.ConsumerConnector;import kafka.message.MessageAndMetadata;import kafka.serializer.StringDecoder;import kafka.utils.VerifiableProperties;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Properties;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;/** * kafka消费者基础，完成了kafka连接、启动、解析kafka数据，并定义了业务处理规范 * * Created by huxin on 2016/12/20. */public abstract class AbstractKafkaConsumer {    // 日志    private final static Logger logger = LoggerFactory.getLogger(AbstractKafkaConsumer.class);    // kafka消息者连接器    private static ConsumerConnector consumer;    // 线程池    private ExecutorService executor;    // 配置文件信息    private PropertiesLoader propertiesLoader = null;    /**     * 初始化kafka消费者连接器     *     * @return     */    public ConsumerConnector getConsumerConnector(String file) {        consumer = kafka.consumer.Consumer.createJavaConsumerConnector(createConsumerConfig(file));        return consumer;    }    /**     * kafka消息进程     */    public void run(String file) {        // 初始化消息者        consumer =  getConsumerConnector (file);        //消费kafka的消息,进程数不能大于kafka集群的数目，否则造成偏移不齐        int thread = Integer.valueOf(propertiesLoader.getProperty("thread") != null?propertiesLoader.getProperty("thread") : "1");        String topic = propertiesLoader.getProperty("topic");        // 键与值都声明为字符串解码        StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());        StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();        topicCountMap.put(topic, thread);        Map<String, List<KafkaStream<String, String>>> consumerMap = consumer.createMessageStreams(topicCountMap, keyDecoder, valueDecoder);        List<KafkaStream<String, String>> streams = consumerMap.get(topic);        try {            executor = Executors.newFixedThreadPool(thread);            int threadNumber = 0;            // 遍历消息，该处为阻塞队列，如果没有消息会一直等待，直到有消息            for (final KafkaStream<String, String> kafkaStream : streams) {                logger.info("creating thread {}", threadNumber);                new Thread(new Runnable() {                    @Override                    public void run() {                        ConsumerIterator<String, String> it = kafkaStream.iterator();                        while (it.hasNext()) {                            MessageAndMetadata<String, String> data = it.next();                            String msg = data.message();                            try {                                if (null != msg && msg.trim().length() > 0) {                                    // 业务处理                                    doBusiness(msg);                                }                                consumer.commitOffsets();                            } catch (Exception e) {                                logger.error("doing message fail! message is :" + msg + ",   mappings.com.ai.ecsite.dao.cache is :" + e);                            }                        }                    }                }).start();                logger.debug("已经启动线程{}", threadNumber);                threadNumber++;            }            logger.warn("exit kafka consumer");            System.in.read();        } catch (Exception e) {            logger.warn("启动consumer线程失败，错误信息：{}", e.getMessage());        }    }    /**     * kafka消费消息时的业务处理类，此方法需要被重写     */    public abstract void doBusiness(String msg) ;    /**     * 关闭消息者与连接     */    public void shutdown (){        // 关闭消费者        if (consumer != null){            consumer.shutdown();        }        if (executor != null){            executor.shutdown();;        }    }    /**     * 创建kafka所需的配置文件     *     * @param     * @return     * @author huxin     * @create 2016/11/24     * @version V1.0.0     */    private ConsumerConfig createConsumerConfig(String file) {        // 加载properties信息        propertiesLoader = new PropertiesLoader(file);        Properties props = new Properties();        props.put("zookeeper.connect", propertiesLoader.getProperty("zookeeper.connect"));        props.put("group.id", propertiesLoader.getProperty("group.id"));        props.put("zookeeper.session.timeout.ms", propertiesLoader.getProperty("zookeeper.session.timeout.ms"));        props.put("zookeeper.sync.time.ms", propertiesLoader.getProperty("zookeeper.sync.time.ms"));        props.put("auto.commit.interval.ms", propertiesLoader.getProperty("auto.commit.interval.ms"));        props.put("auto.commit.enable", propertiesLoader.getProperty("auto.commit.enable"));        props.put("auto.offset.reset", propertiesLoader.getProperty("auto.offset.reset"));        return new ConsumerConfig(props);    }}

0 0