kafka Java实例
来源:互联网 发布:乐高机器人的所有编程 编辑:程序博客网 时间:2024/05/22 03:34
import java.util.*;import kafka.javaapi.producer.Producer;import kafka.producer.KeyedMessage;import kafka.producer.Partitioner;import kafka.producer.ProducerConfig;import kafka.utils.VerifiableProperties;public class TestProducer { class SimplePartitioner implements Partitioner { public SimplePartitioner (VerifiableProperties props) { } public int partition(Object key, int a_numPartitions) { int partition = 0; String stringKey = (String) key; int offset = stringKey.lastIndexOf('.'); if (offset > 0) { partition = Integer.parseInt( stringKey.substring(offset+1)) % a_numPartitions; } return partition; } } public static void main(String[] args) { long events =30; Random rnd = new Random(); Properties props = new Properties(); props.put("metadata.broker.list", "kafka1:9092,kafka2:9092,kafka3:9092"); props.put("serializer.class", "kafka.serializer.StringEncoder"); props.put("partitioner.class", "com.xf.kafa.TestProducer.SimplePartitioner"); props.put("request.required.acks", "1"); ProducerConfig config = new ProducerConfig(props); //The first is the type of the Partition key, the second the type of the message. Producer<String, String> producer = new Producer<String, String>(config); for (long nEvents = 0; nEvents < events; nEvents++) { long runtime = new Date().getTime(); String ip = "192.168.2." + rnd.nextInt(255); String msg = runtime + ",www.example.com," + ip; //参数分别为topic,IP as the partition key,msg为消息 KeyedMessage<String, String> data = new KeyedMessage<String, String>("page_visits", ip, msg); producer.send(data); } producer.close(); }}
public class ConsumerGroupExample { private final ConsumerConnector consumer; private final String topic; private ExecutorService executor; class ConsumerTest implements Runnable { private KafkaStream m_stream; //标识线程名称,可以使用Thread.getCurrentTreandName()代替 private int m_threadNumber; public ConsumerTest(KafkaStream a_stream, int a_threadNumber) { m_threadNumber = a_threadNumber; m_stream = a_stream; } public void run() { ConsumerIterator<byte[], byte[]> it = m_stream.iterator(); //有趣的是 while (it.hasNext())部分. 基于此处会不停的读取topic的信息,除非你停止它。 while (it.hasNext()) System.out.println("Thread " + m_threadNumber + ": " + new String(it.next().message())); System.out.println("Shutting down Thread: " + m_threadNumber); } } public ConsumerGroupExample(String a_zookeeper, String a_groupId, String a_topic) { consumer = kafka.consumer.Consumer.createJavaConsumerConnector( createConsumerConfig(a_zookeeper, a_groupId)); this.topic = a_topic; } public void shutdown() { if (consumer != null) consumer.shutdown(); if (executor != null) executor.shutdown(); try { if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) { System.out.println("Timed out waiting for consumer threads to shut down, exiting uncleanly"); } } catch (InterruptedException e) { System.out.println("Interrupted during shutdown, exiting uncleanly"); } } public void run(int a_numThreads) { /** * First we create a Map that tells Kafka how many threads we are providing for which topics. * The consumer.createMessageStreams is how we pass this information to Kafka. * The return is a map of KafkaStream to listen on for each topic. * (Note here we only asked Kafka for a single Topic but we could have asked for multiple by * adding another element to the Map.) Finally we create the thread pool and pass a new ConsumerTest object to each thread as our business logic. */ //首先我们创建一个map,map能告诉kafka我们为每个topic启动多少个线程 Map<String, Integer> topicCountMap = new HashMap<String, Integer>(); topicCountMap.put(topic, new Integer(a_numThreads)); //consumer.createMessageStreams是我们把信息传递给kafka的方式,返回的是 a map of KafkaStream,来监听每个topic //注意在这我们仅仅向kafka请求一个topic,但是我们可以通过向map添加另外一个元素来请求多个topic Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap); List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topic); // now launch all the threads executor = Executors.newFixedThreadPool(a_numThreads); // now create an object to consume the messages int threadNumber = 0; for (final KafkaStream stream : streams) { //最后我们创建线程池,并且传递ConsumerTest线程给线程池 executor.submit(new ConsumerTest(stream, threadNumber)); threadNumber++; } } private static ConsumerConfig createConsumerConfig(String a_zookeeper, String a_groupId) { Properties props = new Properties(); props.put("zookeeper.connect", a_zookeeper); props.put("group.id", a_groupId); props.put("zookeeper.session.timeout.ms", "400"); props.put("zookeeper.sync.time.ms", "200"); props.put("auto.commit.interval.ms", "1000"); return new ConsumerConfig(props); } public static void main(String[] args) { String zooKeeper = "192.168.8.121:2181,192.168.8.122:2181,192.168.8.123:2181"; String groupId = "test"; String topic = "page_visits"; int threads = 1; ConsumerGroupExample example = new ConsumerGroupExample(zooKeeper, groupId, topic); example.run(threads); try { Thread.sleep(1000000); } catch (InterruptedException ie) { } example.shutdown(); }}
0 0
- kafka Java实例
- Kafka Java Producer实例
- Kafka java实例代码
- kafka的java实例
- 初识Apache Kafka+JAVA程序实例
- java--kafka编程实例--producer和consumer
- kafka 实例
- Kafka 实例
- Kafka 0.11.0.0 producer&consumer java API实例
- kafka java
- 【Apache Kafka】代码实例
- 【Apache Kafka】代码实例
- Kafka编程实例
- Kafka入门实例
- kafka low levle实例
- Spark+kafka+SparkStreaming实例
- kafka spring 实例
- flume-kafka 实例 详细
- 再记看牡丹
- ORACLE连接 (+)和Join总
- NYIST 811变态最大值
- 高并发带来的重复提交问题解决方案
- 第5周 经典小程序2题 C语言实现的猜拳游戏(剪子锤子布)
- kafka Java实例
- PHP中的$this用法
- 从零开始变大牛
- hibernate shards分库的实现
- hdu 5665 Lucky(简单题)(bestcoder#80 1000)
- 下载网页上的各种视频只需四步
- dp 16
- mysql数据库导入导出
- 实现带记忆功能的AutoCompleteTextView