kafka 实例

1、produce

import java.util.{Date, Properties}import kafka.producer.{KeyedMessage, Producer, ProducerConfig}import scala.util.Randomobject ScalaProducerExample extends App {  val events = args(0).toInt  val topic = args(1)  val brokers = args(2)  val rnd = new Random()  val props = new Properties()  props.put("metadata.broker.list", brokers)  props.put("serializer.class", "kafka.serializer.StringEncoder")  props.put("producer.type", "async")  val config = new ProducerConfig(props)  val producer = new Producer[String, String](config)  val t = System.currentTimeMillis()  for (nEvents <- Range(0, events)) {    val runtime = new Date().getTime();    val ip = "192.168.210.16" + rnd.nextInt(255);    val msg = runtime + "," + nEvents + ",www.example.com," + ip;    val data = new KeyedMessage[String, String](topic, ip, msg);    producer.send(data);  }  System.out.println("sent per second: " + events * 1000 / (System.currentTimeMillis() - t));  producer.close();}

2、消费者

import java.util.Propertiesimport java.util.concurrent._import kafka.utils.Loggingimport kafka.consumer.{Consumer, ConsumerConfig, KafkaStream}class ScalaConsumerExample(val zookeeper: String,                           val groupId: String,                           val topic: String,                           val delay: Long) extends Logging {  val config = createConsumerConfig(zookeeper, groupId)  val consumer = Consumer.create(config)  var executor: ExecutorService = null  def shutdown() = {    if (consumer != null)      consumer.shutdown();    if (executor != null)      executor.shutdown();  }  def createConsumerConfig(zookeeper: String, groupId: String): ConsumerConfig = {    val props = new Properties()    props.put("zookeeper.connect", zookeeper);    props.put("group.id", groupId);    props.put("auto.offset.reset", "largest");    props.put("zookeeper.session.timeout.ms", "400");    props.put("zookeeper.sync.time.ms", "200");    props.put("auto.commit.interval.ms", "1000");    val config = new ConsumerConfig(props)    config  }  def run(numThreads: Int) = {    val topicCountMap = Map(topic -> numThreads)    val consumerMap = consumer.createMessageStreams(topicCountMap);    val streams = consumerMap.get(topic).get;    executor = Executors.newFixedThreadPool(numThreads);    var threadNumber = 0;    for (stream <- streams) {      executor.submit(new ScalaConsumerTest(stream, threadNumber, delay))      threadNumber += 1    }  }}object ScalaConsumerExample extends App {  val example = new ScalaConsumerExample(args(0), args(1), args(2),args(4).toLong)  example.run(args(3).toInt)}class ScalaConsumerTest(val stream: KafkaStream[Array[Byte], Array[Byte]], val threadNumber: Int, val delay: Long) extends Logging with Runnable {  def run {    val it = stream.iterator()    while (it.hasNext()) {      val msg = new String(it.next().message());      System.out.println(System.currentTimeMillis() + ",Thread " + threadNumber + ": " + msg);    }    System.out.println("Shutting down Thread: " + threadNumber);  }}

3、运行方式

（1）使用maven打成jar包

1）消费者

java -cp /opt/app/kafka_2.10-0.8.2.1/test/KafkaDemo-1.0-SNAPSHOT.jar:/opt/app/kafka_2.10-0.8.2.1/libs/kafka_2.10-0.8.2.1.jar:/opt/app/scala-2.10.5/lib/scala-library.jar:/opt/app/kafka_2.10-0.8.2.1/libs/log4j-1.2.16.jar:/opt/app/kafka_2.10-0.8.2.1/libs/kafka-clients-0.8.2.1.jar:/opt/app/kafka_2.10-0.8.2.1/libs/zkclient-0.3.jar:/opt/app/kafka_2.10-0.8.2.1/libs/metrics-core-2.2.0.jar:/opt/app/kafka_2.10-0.8.2.1/libs/slf4j-log4j12-1.6.1.jar:/opt/app/kafka_2.10-0.8.2.1/libs/slf4j-api-1.7.6.jar:/opt/app/kafka_2.10-0.8.2.1/libs/zookeeper-3.4.6.jar com.colobu.kafka.ScalaProducerExample 1 mykafka1 192.168.210.16:9092
2）生产者
java -cp /opt/app/kafka_2.10-0.8.2.1/test/KafkaDemo-1.0-SNAPSHOT.jar:/opt/app/kafka_2.10-0.8.2.1/libs/kafka_2.10-0.8.2.1.jar:/opt/app/scala-2.10.5/lib/scala-library.jar:/opt/app/kafka_2.10-0.8.2.1/libs/log4j-1.2.16.jar:/opt/app/kafka_2.10-0.8.2.1/libs/kafka-clients-0.8.2.1.jar:/opt/app/kafka_2.10-0.8.2.1/libs/zkclient-0.3.jar:/opt/app/kafka_2.10-0.8.2.1/libs/metrics-core-2.2.0.jar:/opt/app/kafka_2.10-0.8.2.1/libs/slf4j-log4j12-1.6.1.jar:/opt/app/kafka_2.10-0.8.2.1/libs/slf4j-api-1.7.6.jar:/opt/app/kafka_2.10-0.8.2.1/libs/zookeeper-3.4.6.jar com.colobu.kafka.ScalaConsumerExample 192.168.210.16:2181 group1 mykafka1 1 0

一定要加载第三方包，否则报类找不到错误