kafka+java 伪分布式 之二

对上一篇 kafka+java 伪分布式有生产者和消费者 ，接下来对High Level Consumer做处理 如下：

先看代码 ： 

package com.xiefg.kafka; import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Properties;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.TimeUnit;import kafka.consumer.Consumer;import kafka.consumer.ConsumerConfig;import kafka.consumer.KafkaStream;import kafka.javaapi.consumer.ConsumerConnector;import com.xiefg.conf.ConfigureConstant.KafkaProperties;import com.xiefg.util.PropertiesUtils; /** *   * @ClassName: KafkaConsumer  * @Description: 消费者实例  * @author Comsys-xiefg  * @date 2017年2月5日 上午9:50:48  * */public class KafkaConsumer {private static ConsumerConnector consumer=null;private static  ExecutorService executor;    private static final int THREAD_AMOUNT = 2;//     public static void main(String[] args) {           //1、获取配置    Properties props=getPros();    //2、 创建消费者         consumer = CreateConsumer(props);            Map<String, Integer> topicCountMap = new HashMap<String, Integer>();        //每个topic使用多少个kafkastream读取, 多个consumer        topicCountMap.put(KafkaProperties.TOPIC, THREAD_AMOUNT);        //可以读取多个topic//      topicCountMap.put(TOPIC2, 1);                 Map<String, List<KafkaStream<byte[], byte[]>>> msgStreams = consumer.createMessageStreams(topicCountMap);        List<KafkaStream<byte[], byte[]>> msgStreamList = msgStreams.get(KafkaProperties.TOPIC);                 //使用ExecutorService来调度线程         executor = Executors.newFixedThreadPool(THREAD_AMOUNT);//线程数不要多于分区数        for (int i = 0; i < msgStreamList.size(); i++) {            KafkaStream<byte[], byte[]> kafkaStream = msgStreamList.get(i);            executor.submit(new HanldMessageThread(kafkaStream, i));        }        try {            Thread.sleep(20000);        } catch (InterruptedException e) {            e.printStackTrace();        }    //关闭consumer        closeConsumer();    }     /***      *      * @Title: closeConsumer     * @Description: 关闭consumer     * @return void    返回类型     * @throws      */private static void closeConsumer() {    if (consumer != null) {        consumer.shutdown();    }    if (executor != null) {        executor.shutdown();    }    //在shutdown之后，等待了5秒钟，给consumer线程时间来处理完kafka stream里保留的消息    try {        if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) {            System.out.println("Timed out waiting for consumer threads to shut down, exiting uncleanly");        }    } catch (InterruptedException e) {        System.out.println("Interrupted during shutdown, exiting uncleanly");    }}   /**    *    * @Title: CreateConsumer   * @Description: 创建消费者   * @param props   * @return    设定文件   * @return ConsumerConnector    返回类型   * @throws    */private static ConsumerConnector CreateConsumer(Properties props) {ConsumerConnector consumer = Consumer.createJavaConsumerConnector(new ConsumerConfig(props));return consumer;}        /**     *     * @Title: getPros    * @Description: 从配置文件中获取相关配置    * @return    设定文件    * @return Properties    返回类型    * @throws     */public static Properties getPros(){Properties props = new Properties();        props.put("zookeeper.connect", PropertiesUtils.getPropertiesValue(KafkaProperties.ZK));        props.put("group.id", PropertiesUtils.getPropertiesValue(KafkaProperties.GROUP_ID));        props.put("zookeeper.session.timeout.ms", PropertiesUtils.getPropertiesValue(KafkaProperties.SESSION_TIMEOUT));        props.put("zookeeper.sync.time.ms", PropertiesUtils.getPropertiesValue(KafkaProperties.SYNC_TIME));        props.put("auto.commit.interval.ms", PropertiesUtils.getPropertiesValue(KafkaProperties.INTERVAL));return props;} }

package com.xiefg.kafka;import kafka.consumer.ConsumerIterator;import kafka.consumer.KafkaStream;/** *   * @ClassName: HanldMessageThread  * @Description: 具体处理message的线程  * @author Comsys-xiefg  * @date 2017年2月5日 上午10:46:05  * */class HanldMessageThread implements Runnable {     private KafkaStream<byte[], byte[]> kafkaStream = null;    private int num = 0;         public HanldMessageThread(KafkaStream<byte[], byte[]> kafkaStream, int num) {        super();        this.kafkaStream = kafkaStream;        this.num = num;    }     public void run() {        ConsumerIterator<byte[], byte[]> iterator = kafkaStream.iterator();        while(iterator.hasNext()) {            String message = new String(iterator.next().message());            System.out.println("Thread-->: " + num + ", message: " + message);        }    }     }

package com.xiefg.util;import java.io.BufferedInputStream;import java.io.FileInputStream;import java.io.InputStream;import java.util.PropertyResourceBundle;import java.util.ResourceBundle;import com.xiefg.conf.ConfigureConstant.KafkaProperties;/*** *   * @ClassName: PropertiesUtils  * @Description: TODO  * @author Comsys-xiefg  * @date 2017年2月5日 下午2:41:52  * */public class PropertiesUtils {private static ResourceBundle resources=null;static{InputStream in;try {String config_path = System.getProperty("user.dir") + "/config/system.properties";  in = new BufferedInputStream(new FileInputStream(config_path));resources = new PropertyResourceBundle(in);} catch (Exception e) {e.printStackTrace();}  }   /**     * 获取指定属性值     * @param property 属性名     * @return     */    public static String getPropertiesValue(String property) {        String val = "";        try {            val = resources.getString(property);        } catch (Exception e) {            // ignore        e.printStackTrace();        }        return val;    }public static void main(String[] args) {System.out.println(PropertiesUtils.getPropertiesValue(KafkaProperties.ZK));}}

package com.xiefg.conf;/** *   * @ClassName: ConfigureConstant  * @Description: kafka 配置常量  * @author Comsys-xiefg  * @date 2017年2月5日 下午2:22:58  * */public class ConfigureConstant {    public interface KafkaProperties {            public final static String ZK = "kafka.zookeeper.connect";        public final static String GROUP_ID = "kafka.group.id";        public final static String SESSION_TIMEOUT = "kafka.zookeeper.session.timeout.ms";        public final static String SYNC_TIME = "kafka.zookeeper.sync.time.ms";        public final static String INTERVAL = "kafka.auto.commit.interval.ms";        //主题        public final static String TOPIC = "test_topic";              }}

配置文件system.properties

kafka.zookeeper.connect=192.168.110.69:2181kafka.group.id=69kafka.zookeeper.session.timeout.ms=40000kafka.zookeeper.sync.time.ms=20000kafka.auto.commit.interval.ms=10000

运行 ProducerTest 类 （上一篇文章中的），然后运行KafkaConsumer 结果如下：

2017-02-05 16:07:21,490 INFO [kafka.utils.VerifiableProperties] - Verifying properties2017-02-05 16:07:21,532 INFO [kafka.utils.VerifiableProperties] - Property auto.commit.interval.ms is overridden to 100002017-02-05 16:07:21,533 INFO [kafka.utils.VerifiableProperties] - Property group.id is overridden to 692017-02-05 16:07:21,533 INFO [kafka.utils.VerifiableProperties] - Property zookeeper.connect is overridden to 192.168.170.69:21812017-02-05 16:07:21,533 INFO [kafka.utils.VerifiableProperties] - Property zookeeper.session.timeout.ms is overridden to 400002017-02-05 16:07:21,533 INFO [kafka.utils.VerifiableProperties] - Property zookeeper.sync.time.ms is overridden to 200002017-02-05 16:07:25,052 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], Connecting to zookeeper instance at 192.168.170.69:21812017-02-05 16:07:25,120 INFO [org.I0Itec.zkclient.ZkEventThread] - Starting ZkClient event thread.2017-02-05 16:07:25,150 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT2017-02-05 16:07:25,150 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:host.name=192.168.10.892017-02-05 16:07:25,150 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:java.version=1.7.0_032017-02-05 16:07:25,150 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:java.vendor=Oracle Corporation2017-02-05 16:07:25,150 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:java.home=D:\Java\jdk1.7.0_03\jre2017-02-05 16:07:25,151 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:java.class.path=D:\hadoop\workspace\kafka\target\classes;C:\Users\xiefg\.m2\repository\org\apache\kafka\kafka_2.10\0.8.2.0\kafka_2.10-0.8.2.0.jar;C:\Users\xiefg\.m2\repository\org\apache\kafka\kafka-clients\0.8.2.0\kafka-clients-0.8.2.0.jar;C:\Users\xiefg\.m2\repository\org\slf4j\slf4j-api\1.7.6\slf4j-api-1.7.6.jar;C:\Users\xiefg\.m2\repository\net\jpountz\lz4\lz4\1.2.0\lz4-1.2.0.jar;C:\Users\xiefg\.m2\repository\org\xerial\snappy\snappy-java\1.1.1.6\snappy-java-1.1.1.6.jar;C:\Users\xiefg\.m2\repository\com\yammer\metrics\metrics-core\2.2.0\metrics-core-2.2.0.jar;C:\Users\xiefg\.m2\repository\org\scala-lang\scala-library\2.10.4\scala-library-2.10.4.jar;C:\Users\xiefg\.m2\repository\org\apache\zookeeper\zookeeper\3.4.6\zookeeper-3.4.6.jar;C:\Users\xiefg\.m2\repository\org\slf4j\slf4j-log4j12\1.6.1\slf4j-log4j12-1.6.1.jar;C:\Users\xiefg\.m2\repository\log4j\log4j\1.2.16\log4j-1.2.16.jar;C:\Users\xiefg\.m2\repository\jline\jline\0.9.94\jline-0.9.94.jar;C:\Users\xiefg\.m2\repository\io\netty\netty\3.7.0.Final\netty-3.7.0.Final.jar;C:\Users\xiefg\.m2\repository\net\sf\jopt-simple\jopt-simple\3.2\jopt-simple-3.2.jar;C:\Users\xiefg\.m2\repository\com\101tec\zkclient\0.3\zkclient-0.3.jar2017-02-05 16:07:25,151 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:java.library.path=D:\Java\jdk1.7.0_03\bin;C:\Windows\Sun\Java\bin;C:\Windows\system32;C:\Windows;E:/MyEclipse/binary/com.sun.java.jdk7.win32.x86_1.7.0.u45/jre/bin/client;E:/MyEclipse/binary/com.sun.java.jdk7.win32.x86_1.7.0.u45/jre/bin;E:/MyEclipse/binary/com.sun.java.jdk7.win32.x86_1.7.0.u45/jre/lib/i386;C:\Program Files\Common Files\NetSarang;C:\Program Files\NVIDIA Corporation\PhysX\Common;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files\Intel\OpenCL SDK\2.0\bin\x86; D:\Java\jdk1.7.0_03\bin;D:\Java\jdk1.7.0_03\jre\bin;D:\hadoop-2.6.1-64/bin;D:\Rational\common;C:\Python33;D:\instantclient_11_2;C:\Program Files\MySQL\MySQL Server 5.5\bin;.2017-02-05 16:07:25,151 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:java.io.tmpdir=C:\Users\xiefg\AppData\Local\Temp\2017-02-05 16:07:25,151 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:java.compiler=<NA>2017-02-05 16:07:25,151 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:os.name=Windows 72017-02-05 16:07:25,151 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:os.arch=x862017-02-05 16:07:25,151 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:os.version=6.12017-02-05 16:07:25,151 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:user.name=xiefg2017-02-05 16:07:25,151 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:user.home=C:\Users\xiefg2017-02-05 16:07:25,151 INFO [org.apache.zookeeper.ZooKeeper] - Client environment:user.dir=D:\hadoop\workspace\kafka2017-02-05 16:07:25,152 INFO [org.apache.zookeeper.ZooKeeper] - Initiating client connection, connectString=192.168.170.69:2181 sessionTimeout=40000 watcher=org.I0Itec.zkclient.ZkClient@10974382017-02-05 16:07:25,446 INFO [org.apache.zookeeper.ClientCnxn] - Opening socket connection to server 192.168.170.69/192.168.170.69:2181. Will not attempt to authenticate using SASL (unknown error)2017-02-05 16:07:25,447 INFO [org.apache.zookeeper.ClientCnxn] - Socket connection established to 192.168.170.69/192.168.170.69:2181, initiating session2017-02-05 16:07:25,457 INFO [org.apache.zookeeper.ClientCnxn] - Session establishment complete on server 192.168.170.69/192.168.170.69:2181, sessionid = 0x15a0c0833ba0006, negotiated timeout = 400002017-02-05 16:07:25,459 INFO [org.I0Itec.zkclient.ZkClient] - zookeeper state changed (SyncConnected)2017-02-05 16:07:25,520 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], starting auto committer every 10000 ms2017-02-05 16:07:25,626 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], begin registering consumer 69_xiefg-PC-1486282045030-7d9770a7 in ZK2017-02-05 16:07:25,774 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], end registering consumer 69_xiefg-PC-1486282045030-7d9770a7 in ZK2017-02-05 16:07:25,825 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], starting watcher executor thread for consumer 69_xiefg-PC-1486282045030-7d9770a72017-02-05 16:07:25,919 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], begin rebalancing consumer 69_xiefg-PC-1486282045030-7d9770a7 try #02017-02-05 16:07:26,448 INFO [kafka.consumer.ConsumerFetcherManager] - [ConsumerFetcherManager-1486282045496] Stopping leader finder thread2017-02-05 16:07:26,448 INFO [kafka.consumer.ConsumerFetcherManager] - [ConsumerFetcherManager-1486282045496] Stopping all fetchers2017-02-05 16:07:26,451 INFO [kafka.consumer.ConsumerFetcherManager] - [ConsumerFetcherManager-1486282045496] All connections stopped2017-02-05 16:07:26,452 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], Cleared all relevant queues for this fetcher2017-02-05 16:07:26,454 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], Cleared the data chunks in all the consumer message iterators2017-02-05 16:07:26,455 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], Committing all offsets after clearing the fetcher queues2017-02-05 16:07:26,456 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], Releasing partition ownership2017-02-05 16:07:26,538 INFO [kafka.consumer.RangeAssignor] - Consumer 69_xiefg-PC-1486282045030-7d9770a7 rebalancing the following partitions: ArrayBuffer(0, 1) for topic test_topic with consumers: List(69_xiefg-PC-1486282045030-7d9770a7-0, 69_xiefg-PC-1486282045030-7d9770a7-1)2017-02-05 16:07:26,540 INFO [kafka.consumer.RangeAssignor] - 69_xiefg-PC-1486282045030-7d9770a7-0 attempting to claim partition 02017-02-05 16:07:26,541 INFO [kafka.consumer.RangeAssignor] - 69_xiefg-PC-1486282045030-7d9770a7-1 attempting to claim partition 12017-02-05 16:07:26,601 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], 69_xiefg-PC-1486282045030-7d9770a7-0 successfully owned partition 0 for topic test_topic2017-02-05 16:07:26,603 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], 69_xiefg-PC-1486282045030-7d9770a7-1 successfully owned partition 1 for topic test_topic2017-02-05 16:07:26,655 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], Consumer 69_xiefg-PC-1486282045030-7d9770a7 selected partitions : test_topic:0: fetched offset = 50: consumed offset = 50,test_topic:1: fetched offset = 50: consumed offset = 502017-02-05 16:07:26,692 INFO [kafka.consumer.ConsumerFetcherManager$LeaderFinderThread] - [69_xiefg-PC-1486282045030-7d9770a7-leader-finder-thread], Starting 2017-02-05 16:07:26,694 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], end rebalancing consumer 69_xiefg-PC-1486282045030-7d9770a7 try #02017-02-05 16:07:26,714 INFO [kafka.utils.VerifiableProperties] - Verifying properties2017-02-05 16:07:26,714 INFO [kafka.utils.VerifiableProperties] - Property client.id is overridden to 692017-02-05 16:07:26,714 INFO [kafka.utils.VerifiableProperties] - Property metadata.broker.list is overridden to 192.168.170.69:90922017-02-05 16:07:26,715 INFO [kafka.utils.VerifiableProperties] - Property request.timeout.ms is overridden to 300002017-02-05 16:07:26,731 INFO [kafka.client.ClientUtils$] - Fetching metadata from broker id:69,host:192.168.170.69,port:9092 with correlation id 0 for 1 topic(s) Set(test_topic)2017-02-05 16:07:26,734 INFO [kafka.producer.SyncProducer] - Connected to 192.168.170.69:9092 for producing2017-02-05 16:07:26,752 INFO [kafka.producer.SyncProducer] - Disconnecting from 192.168.170.69:90922017-02-05 16:07:26,821 INFO [kafka.consumer.ConsumerFetcherThread] - [ConsumerFetcherThread-69_xiefg-PC-1486282045030-7d9770a7-0-69], Starting 2017-02-05 16:07:26,824 INFO [kafka.consumer.ConsumerFetcherManager] - [ConsumerFetcherManager-1486282045496] Added fetcher for partitions ArrayBuffer([[test_topic,0], initOffset 50 to broker id:69,host:192.168.170.69,port:9092] , [[test_topic,1], initOffset 50 to broker id:69,host:192.168.170.69,port:9092] )Thread-->: 0, message: xiefg.org0=2017年02月05日 16:07:09 758Thread-->: 1, message: xiefg.org1=2017年02月05日 16:07:11 467Thread-->: 0, message: xiefg.org2=2017年02月05日 16:07:11 471Thread-->: 1, message: xiefg.org3=2017年02月05日 16:07:11 473Thread-->: 0, message: xiefg.org4=2017年02月05日 16:07:11 476Thread-->: 1, message: xiefg.org5=2017年02月05日 16:07:11 479Thread-->: 0, message: xiefg.org6=2017年02月05日 16:07:11 482Thread-->: 0, message: xiefg.org8=2017年02月05日 16:07:11 487Thread-->: 1, message: xiefg.org7=2017年02月05日 16:07:11 484Thread-->: 1, message: xiefg.org9=2017年02月05日 16:07:11 4892017-02-05 16:07:46,697 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], ZKConsumerConnector shutting down2017-02-05 16:07:46,749 INFO [kafka.consumer.ConsumerFetcherManager] - [ConsumerFetcherManager-1486282045496] Stopping leader finder thread2017-02-05 16:07:46,751 INFO [kafka.consumer.ConsumerFetcherManager$LeaderFinderThread] - [69_xiefg-PC-1486282045030-7d9770a7-leader-finder-thread], Shutting down2017-02-05 16:07:46,752 INFO [kafka.consumer.ConsumerFetcherManager$LeaderFinderThread] - [69_xiefg-PC-1486282045030-7d9770a7-leader-finder-thread], Stopped 2017-02-05 16:07:46,752 INFO [kafka.consumer.ConsumerFetcherManager$LeaderFinderThread] - [69_xiefg-PC-1486282045030-7d9770a7-leader-finder-thread], Shutdown completed2017-02-05 16:07:46,752 INFO [kafka.consumer.ConsumerFetcherManager] - [ConsumerFetcherManager-1486282045496] Stopping all fetchers2017-02-05 16:07:46,753 INFO [kafka.consumer.ConsumerFetcherThread] - [ConsumerFetcherThread-69_xiefg-PC-1486282045030-7d9770a7-0-69], Shutting down2017-02-05 16:07:46,769 INFO [kafka.consumer.SimpleConsumer] - Reconnect due to socket error: java.nio.channels.ClosedByInterruptException2017-02-05 16:07:46,769 INFO [kafka.consumer.ConsumerFetcherThread] - [ConsumerFetcherThread-69_xiefg-PC-1486282045030-7d9770a7-0-69], Stopped 2017-02-05 16:07:46,769 INFO [kafka.consumer.ConsumerFetcherThread] - [ConsumerFetcherThread-69_xiefg-PC-1486282045030-7d9770a7-0-69], Shutdown completed2017-02-05 16:07:46,770 INFO [kafka.consumer.ConsumerFetcherManager] - [ConsumerFetcherManager-1486282045496] All connections stopped2017-02-05 16:07:46,785 INFO [org.I0Itec.zkclient.ZkEventThread] - Terminate ZkClient event thread.2017-02-05 16:07:46,802 INFO [org.apache.zookeeper.ZooKeeper] - Session: 0x15a0c0833ba0006 closed2017-02-05 16:07:46,803 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], ZKConsumerConnector shutdown completed in 105 ms2017-02-05 16:07:46,803 INFO [org.apache.zookeeper.ClientCnxn] - EventThread shut down2017-02-05 16:07:46,836 INFO [kafka.consumer.ZookeeperConsumerConnector] - [69_xiefg-PC-1486282045030-7d9770a7], stopping watcher executor thread for consumer 69_xiefg-PC-1486282045030-7d9770a7

可以看到在shutdown之后，等待了5秒钟，给consumer线程时间来处理完kafka stream里保留的消息。

为什么要用High Level Consumer

     某些场景，从Kafka中读取消息的逻辑不处理消息的offset，仅仅是获取消息数据。High Level Consumer就提供了这种功能。首先要知道的是，High Level Consumer在ZooKeeper上保存最新的offset（从指定的分区中读取）。这个offset基于consumer group名存储。Consumer group名在Kafka集群上是全局性的，在启动新的consumer group的时候要小心集群上没有关闭的consumer。当一个consumer线程启动了，Kafka会将它加入到相同的topic下的相同consumer group里，并且触发重新分配。在重新分配时，Kafka将partition分配给consumer，有可能会移动一个partition给另一个consumer。如果老的、新的处理逻辑同时存在，有可能将一些消息传递到了老的consumer上。

设计High Level Consumer

      使用High LevelConsumer，它是多线程的。消费者线程的数量跟topic的partition数量有关，它们之间有一些特定的规则：

        1、如果线程数量大于主题的分区数量，一些线程将得不到任何消息。

        2、如果分区数大于线程数，一些线程将得到多个分区的消息。

        3、如果一个线程处理多个分区的消息，它接收到消息的顺序是不能保证的。比如，先从分区10获取了5条消息，从分区11获取了6条消息，然后从分区10获取了5条，紧接着又从分区10获取了5条，虽然分区2还有消息。

添加更多了同consumer group的consumer将触发Kafka重新分配，某个分区本来分配给a线程，当从新分配后，有可能分配给了b线程。

关闭消费组和错误处理

        Kafka不会再每次读取消息后马上更新zookeeper上的offset，而是等待一段时间。由于这种延迟，有可能消费者读取了一条消息，但没有更新offset。所以，当客户端关闭或崩溃后，从新启动时有些消息重复读取了。另外，broker宕机或其他原因导致更换了partition的leader，也会导致消息重复读取。

为了避免这种问题，你应该提供一个平滑的关闭方式，而不是直接kill 掉。

上面的是解释是从  https://cwiki.apache.org/confluence/display/KAFKA/Consumer+Group+Example   上翻译的 不对的地方还请指正