kafka的java实例

1、依赖的jar包

 <properties>        <kafka.version>0.10.2.0</kafka.version>    </properties> <dependency>        <groupId>org.apache.kafka</groupId>        <artifactId>kafka-clients</artifactId>        <version>${kafka.version}</version>    </dependency>    <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka_2.11 -->    <dependency>        <groupId>org.apache.kafka</groupId>        <artifactId>kafka_2.11</artifactId>        <version>${kafka.version}</version>    </dependency>

2、生产者

package com.kafka.demo;import org.apache.kafka.clients.producer.KafkaProducer;import org.apache.kafka.clients.producer.Producer;import org.apache.kafka.clients.producer.ProducerRecord;import java.text.SimpleDateFormat;import java.util.Date;import java.util.Properties;/** * Created by liyongguan on 2017/11/22. */public class KafkaProducerDemo {    /***  broker所在的机器 **/    private final  String BROKER_LIST="172.16.36.112:9092,172.163.6.120:9092,172.16.36.121:9092";    private final  String TOPIC = "test";    public Properties getProperties(){        Properties proper = new Properties();        /* 定义kakfa 服务的地址 */        proper.put("bootstrap.servers",BROKER_LIST);        proper.put("ack","all");        proper.put("retries",0);        proper.put("batch.size",16384);        proper.put("linger.ms",1);        proper.put("buffer.memory",33554432);        proper.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");        proper.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");        return  proper;    }    public void  produceMessage(){        Properties properties = getProperties();        Producer<String,String> producer = new KafkaProducer<String, String>(properties);        String key,value;        for(int i = 1;i<=20;i++){            key = "key-"+i;            value = "value-"+i+"-"+new SimpleDateFormat("HH:mm:ss").format(new Date());            System.out.println("TOPIC: "+TOPIC+",key: "+key+",value: "+value);            producer.send(new ProducerRecord<String, String>(TOPIC,key,value));            try {                Thread.sleep(1000);            }catch (InterruptedException e){                e.printStackTrace();            }        }        producer.close();    }    public static void main(String[] args){        new KafkaProducerDemo().produceMessage();    }}

3、消费者

package com.kafka.demo;import kafka.common.Topic;import org.apache.kafka.clients.consumer.ConsumerRecord;import org.apache.kafka.clients.consumer.ConsumerRecords;import org.apache.kafka.clients.consumer.KafkaConsumer;import org.apache.kafka.common.errors.WakeupException;import java.util.ArrayList;import java.util.Arrays;import java.util.List;import java.util.Properties;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicBoolean;/** * Created by liyongguan on 2017/11/22. */public class KafkaConsumerDemo {    //private  final static String ZOOKEEPER_LIST = "172.16.36.112:2181,172.16.36.120:2181,172.16.36.121:2181";    private  final static String BROKER_LIST="172.16.36.112:9092";    private  final  String TOPIC = "test";    /**     * 连接kafka的配置     * @return     */    public static Properties getProperties(){        Properties properties  = new Properties();        /* 定义kakfa 服务的地址，不需要将所有broker指定上 */        properties.put("bootstrap.servers",BROKER_LIST);        /* 制定consumer group */        properties.put("group.id","test-group");        /* 是否自动确认选项 ,设置不自动提交，采用手动方式*/        properties.put("enable.auto.commit","false");        /* key的序列化类 */        properties.put("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer");        /* value的序列化类 */        properties.put("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer");        return properties;    }    public void consumerMessage(){        /***使用线程池处理***/        int consumerNums = 1;        final ExecutorService executor =Executors.newFixedThreadPool(consumerNums);        final List<KafkaConsumerRunner> kafkaConsumerRunnerList = new ArrayList<KafkaConsumerRunner>();        for(int i = 0;i<consumerNums;i++){            KafkaConsumerRunner kafkaConsumerRunner = new KafkaConsumerRunner(TOPIC);            kafkaConsumerRunnerList.add(kafkaConsumerRunner);            executor.submit(kafkaConsumerRunner);            System.out.println(1234);        }        /*Runtime.getRuntime().addShutdownHook(new Thread(){            public void run(){                for(KafkaConsumerRunner kafkaConsumerRunner:kafkaConsumerRunnerList){                    kafkaConsumerRunner.shutdown();                }                executor.shutdown();                try{                    executor.awaitTermination(1000,TimeUnit.MILLISECONDS);                }catch (InterruptedException e){                    e.printStackTrace();                }            }        });*/    }    /**     * 线程处理--消费者     */    public static  class KafkaConsumerRunner implements Runnable{        private final  String topic;        private final AtomicBoolean  closed = new AtomicBoolean(false);        private  final KafkaConsumer<String,String> consumer;        public KafkaConsumerRunner(String topic){            this.topic = topic;            Properties props = getProperties();            consumer = new KafkaConsumer<String,String>(props);        }        public void handleRecord(ConsumerRecord record){            System.out.println(/*"Thread name: "+Thread.currentThread().getName()+*/"topic:"+record.topic()+" , offset:" + record.offset() + ", key:" + record.key() + ", value:" + record.value());        }        public void run(){            // 批量提交数量            final int minBatchSize = 1;            try{                /* 消费者订阅的topic, 可同时订阅多个 */                consumer.subscribe(Arrays.asList(topic));                List<ConsumerRecord<String, String>> buffer = new ArrayList<ConsumerRecord<String, String>>();                /* 读取数据，读取超时时间为1000ms */                while (!closed.get()){                    ConsumerRecords<String,String> records = consumer.poll(1000);                    for (ConsumerRecord record:records){                        handleRecord(record);                        buffer.add(record);                    }                    if (buffer.size() >= minBatchSize) {                        //这里就是处理成功了然后自己手动提交                        consumer.commitSync();                        buffer.clear();                    }                }            }catch (WakeupException e){                e.printStackTrace();                if(!closed.get())                    throw e;            }            finally {                consumer.close();            }        }        public void shutdown()        {            closed.set(true);            consumer.wakeup();        }    }    public static void main(String[] args){        new KafkaConsumerDemo().consumerMessage();    }}

说明：关于kafka集群的搭建和配置，自行脑补。