c语言使用librdkafka库实现kafka的生产和消费实例

关于librdkafka库的介绍，可以参考kafka的c/c++高性能客户端librdkafka简介，本文使用librdkafka库来进行kafka的简单的生产、消费

一、producer

librdkafka进行kafka生产操作的大致步骤如下：

1、创建kafka配置

rd_kafka_conf_t *rd_kafka_conf_new (void)

2、配置kafka各项参数

rd_kafka_conf_res_t rd_kafka_conf_set (rd_kafka_conf_t *conf,                                       const char *name,                                       const char *value,                                       char *errstr, size_t errstr_size)

3、设置发送回调函数

void rd_kafka_conf_set_dr_msg_cb (rd_kafka_conf_t *conf,                                  void (*dr_msg_cb) (rd_kafka_t *rk,                                  const rd_kafka_message_t *                                  rkmessage,                                  void *opaque))

4、创建producer实例

rd_kafka_t *rd_kafka_new (rd_kafka_type_t type, rd_kafka_conf_t *conf,char *errstr, size_t errstr_size)

5、实例化topic

rd_kafka_topic_t *rd_kafka_topic_new (rd_kafka_t *rk, const char *topic, rd_kafka_topic_conf_t *conf)

6、异步调用将消息发送到指定的topic

int rd_kafka_produce (rd_kafka_topic_t *rkt, int32_t partition,      int msgflags,      void *payload, size_t len,      const void *key, size_t keylen,      void *msg_opaque)

7、阻塞等待消息发送完成

int rd_kafka_poll (rd_kafka_t *rk, int timeout_ms)

8、等待完成producer请求完成

rd_kafka_resp_err_t rd_kafka_flush (rd_kafka_t *rk, int timeout_ms)

9、销毁topic

void rd_kafka_topic_destroy (rd_kafka_topic_t *app_rkt)

10、销毁producer实例

void rd_kafka_destroy (rd_kafka_t *rk)

完整代码如下my_producer.c：

#include <stdio.h>#include <signal.h>#include <string.h>#include "../src/rdkafka.h"static int run = 1;static void stop(int sig){run = 0;fclose(stdin);}/*    每条消息调用一次该回调函数，说明消息是传递成功(rkmessage->err == RD_KAFKA_RESP_ERR_NO_ERROR)    还是传递失败(rkmessage->err != RD_KAFKA_RESP_ERR_NO_ERROR)    该回调函数由rd_kafka_poll()触发，在应用程序的线程上执行 */static void dr_msg_cb(rd_kafka_t *rk,  const rd_kafka_message_t *rkmessage, void *opaque){if(rkmessage->err)fprintf(stderr, "%% Message delivery failed: %s\n", rd_kafka_err2str(rkmessage->err));elsefprintf(stderr,                        "%% Message delivered (%zd bytes, "                        "partition %"PRId32")\n",                        rkmessage->len, rkmessage->partition);        /* rkmessage被librdkafka自动销毁*/}int main(int argc, char **argv){rd_kafka_t *rk;            /*Producer instance handle*/rd_kafka_topic_t *rkt;     /*topic对象*/rd_kafka_conf_t *conf;     /*临时配置对象*/char errstr[512];          char buf[512];             const char *brokers;       const char *topic;         if(argc != 3){fprintf(stderr, "%% Usage: %s <broker> <topic>\n", argv[0]);        return 1;}brokers = argv[1];topic = argv[2];    /* 创建一个kafka配置占位 */conf = rd_kafka_conf_new();    /*创建broker集群*/if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr,sizeof(errstr)) != RD_KAFKA_CONF_OK){fprintf(stderr, "%s\n", errstr);return 1;}    /*设置发送报告回调函数，rd_kafka_produce()接收的每条消息都会调用一次该回调函数     *应用程序需要定期调用rd_kafka_poll()来服务排队的发送报告回调函数*/rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);    /*创建producer实例      rd_kafka_new()获取conf对象的所有权,应用程序在此调用之后不得再次引用它*/rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr));if(!rk){fprintf(stderr, "%% Failed to create new producer:%s\n", errstr);return 1;}    /*实例化一个或多个topics(`rd_kafka_topic_t`)来提供生产或消费，topic    对象保存topic特定的配置，并在内部填充所有可用分区和leader brokers，*/rkt = rd_kafka_topic_new(rk, topic, NULL);if (!rkt){fprintf(stderr, "%% Failed to create topic object: %s\n", rd_kafka_err2str(rd_kafka_last_error()));rd_kafka_destroy(rk);return 1;}    /*用于中断的信号*/signal(SIGINT, stop);fprintf(stderr,                "%% Type some text and hit enter to produce message\n"                "%% Or just hit enter to only serve delivery reports\n"                "%% Press Ctrl-C or Ctrl-D to exit\n");     while(run && fgets(buf, sizeof(buf), stdin)){     size_t len = strlen(buf);     if(buf[len-1] == '\n')     buf[--len] = '\0';     if(len == 0){            /*轮询用于事件的kafka handle，            事件将导致应用程序提供的回调函数被调用            第二个参数是最大阻塞时间，如果设为0，将会是非阻塞的调用*/     rd_kafka_poll(rk, 0);     continue;     }     retry:         /*Send/Produce message.           这是一个异步调用，在成功的情况下，只会将消息排入内部producer队列，           对broker的实际传递尝试由后台线程处理，之前注册的传递回调函数(dr_msg_cb)           用于在消息传递成功或失败时向应用程序发回信号*/     if (rd_kafka_produce(                    /* Topic object */     rkt,                    /*使用内置的分区来选择分区*/     RD_KAFKA_PARTITION_UA,                    /*生成payload的副本*/     RD_KAFKA_MSG_F_COPY,                    /*消息体和长度*/     buf, len,                    /*可选键及其长度*/     NULL, 0,     NULL) == -1){     fprintf(stderr,      "%% Failed to produce to topic %s: %s\n",      rd_kafka_topic_name(rkt),     rd_kafka_err2str(rd_kafka_last_error()));     if (rd_kafka_last_error() == RD_KAFKA_RESP_ERR__QUEUE_FULL){                /*如果内部队列满，等待消息传输完成并retry,                内部队列表示要发送的消息和已发送或失败的消息，                内部队列受限于queue.buffering.max.messages配置项*/     rd_kafka_poll(rk, 1000);     goto retry;     }     }else{     fprintf(stderr, "%% Enqueued message (%zd bytes) for topic %s\n",      len, rd_kafka_topic_name(rkt));     }        /*producer应用程序应不断地通过以频繁的间隔调用rd_kafka_poll()来为        传送报告队列提供服务。在没有生成消息以确定先前生成的消息已发送了其        发送报告回调函数(和其他注册过的回调函数)期间，要确保rd_kafka_poll()        仍然被调用*/     rd_kafka_poll(rk, 0);     }     fprintf(stderr, "%% Flushing final message.. \n");     /*rd_kafka_flush是rd_kafka_poll()的抽象化，     等待所有未完成的produce请求完成，通常在销毁producer实例前完成     以确保所有排列中和正在传输的produce请求在销毁前完成*/     rd_kafka_flush(rk, 10*1000);     /* Destroy topic object */     rd_kafka_topic_destroy(rkt);     /* Destroy the producer instance */     rd_kafka_destroy(rk);     return 0;}

二、consumer

librdkafka进行kafka消费操作的大致步骤如下：

1、创建kafka配置

rd_kafka_conf_t *rd_kafka_conf_new (void)

2、创建kafka topic的配置

rd_kafka_topic_conf_t *rd_kafka_topic_conf_new (void) 

3、配置kafka各项参数

rd_kafka_conf_res_t rd_kafka_conf_set (rd_kafka_conf_t *conf,                                       const char *name,                                       const char *value,                                       char *errstr, size_t errstr_size)

4、配置kafka topic各项参数

rd_kafka_conf_res_t rd_kafka_topic_conf_set (rd_kafka_topic_conf_t *conf,     const char *name,     const char *value,     char *errstr, size_t errstr_size)

5、创建consumer实例

rd_kafka_t *rd_kafka_new (rd_kafka_type_t type, rd_kafka_conf_t *conf, char *errstr, size_t errstr_size)

6、为consumer实例添加brokerlist

int rd_kafka_brokers_add (rd_kafka_t *rk, const char *brokerlist)

7、开启consumer订阅

rd_kafka_subscribe (rd_kafka_t *rk, const rd_kafka_topic_partition_list_t *topics)

8、轮询消息或事件，并调用回调函数

rd_kafka_message_t *rd_kafka_consumer_poll (rd_kafka_t *rk,int timeout_ms)

9、关闭consumer实例

rd_kafka_resp_err_t rd_kafka_consumer_close (rd_kafka_t *rk)

10、释放topic list资源

rd_kafka_topic_partition_list_destroy (rd_kafka_topic_partition_list_t *rktparlist)

11、销毁consumer实例

void rd_kafka_destroy (rd_kafka_t *rk) 

12、等待consumer对象的销毁

int rd_kafka_wait_destroyed (int timeout_ms)

完整代码如下my_consumer.c

#include <string.h>#include <stdlib.h>#include <syslog.h>#include <signal.h>#include <error.h>#include <getopt.h>#include "../src/rdkafka.h"static int run = 1;//`rd_kafka_t`自带一个可选的配置API，如果没有调用API，Librdkafka将会使用CONFIGURATION.md中的默认配置。static rd_kafka_t *rk;static rd_kafka_topic_partition_list_t *topics;static void stop (int sig) {  if (!run)    exit(1);  run = 0;  fclose(stdin); /* abort fgets() */}static void sig_usr1 (int sig) {  rd_kafka_dump(stdout, rk);}/** * 处理并打印已消费的消息 */static void msg_consume (rd_kafka_message_t *rkmessage,       void *opaque) {  if (rkmessage->err) {    if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {      fprintf(stderr,        "%% Consumer reached end of %s [%"PRId32"] "             "message queue at offset %"PRId64"\n",             rd_kafka_topic_name(rkmessage->rkt),             rkmessage->partition, rkmessage->offset);      return;    }    if (rkmessage->rkt)            fprintf(stderr, "%% Consume error for "                    "topic \"%s\" [%"PRId32"] "                    "offset %"PRId64": %s\n",                    rd_kafka_topic_name(rkmessage->rkt),                    rkmessage->partition,                    rkmessage->offset,                    rd_kafka_message_errstr(rkmessage));    else            fprintf(stderr, "%% Consumer error: %s: %s\n",                    rd_kafka_err2str(rkmessage->err),                    rd_kafka_message_errstr(rkmessage));    if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION ||        rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC)          run = 0;    return;  }  fprintf(stdout, "%% Message (topic %s [%"PRId32"], "                      "offset %"PRId64", %zd bytes):\n",                      rd_kafka_topic_name(rkmessage->rkt),                      rkmessage->partition,    rkmessage->offset, rkmessage->len);  if (rkmessage->key_len) {    printf("Key: %.*s\n",             (int)rkmessage->key_len, (char *)rkmessage->key);  }  printf("%.*s\n",           (int)rkmessage->len, (char *)rkmessage->payload);  }/*  init all configuration of kafka */int initKafka(char *brokers, char *group,char *topic){  rd_kafka_conf_t *conf;  rd_kafka_topic_conf_t *topic_conf;  rd_kafka_resp_err_t err;  char tmp[16];  char errstr[512];  /* Kafka configuration */  conf = rd_kafka_conf_new();  //quick termination  snprintf(tmp, sizeof(tmp), "%i", SIGIO);  rd_kafka_conf_set(conf, "internal.termination.signal", tmp, NULL, 0);  //topic configuration  topic_conf = rd_kafka_topic_conf_new();  /* Consumer groups require a group id */  if (!group)          group = "rdkafka_consumer_example";  if (rd_kafka_conf_set(conf, "group.id", group,                        errstr, sizeof(errstr)) !=      RD_KAFKA_CONF_OK) {          fprintf(stderr, "%% %s\n", errstr);          return -1;  }  /* Consumer groups always use broker based offset storage */  if (rd_kafka_topic_conf_set(topic_conf, "offset.store.method",                              "broker",                              errstr, sizeof(errstr)) !=      RD_KAFKA_CONF_OK) {          fprintf(stderr, "%% %s\n", errstr);          return -1;  }  /* Set default topic config for pattern-matched topics. */  rd_kafka_conf_set_default_topic_conf(conf, topic_conf);  //实例化一个顶级对象rd_kafka_t作为基础容器，提供全局配置和共享状态  rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));  if(!rk){    fprintf(stderr, "%% Failed to create new consumer:%s\n", errstr);    return -1;  }  //Librdkafka需要至少一个brokers的初始化list  if (rd_kafka_brokers_add(rk, brokers) == 0){    fprintf(stderr, "%% No valid brokers specified\n");    return -1;  }  //重定向 rd_kafka_poll()队列到consumer_poll()队列  rd_kafka_poll_set_consumer(rk);  //创建一个Topic+Partition的存储空间(list/vector)  topics = rd_kafka_topic_partition_list_new(1);  //把Topic+Partition加入list  rd_kafka_topic_partition_list_add(topics, topic, -1);  //开启consumer订阅，匹配的topic将被添加到订阅列表中  if((err = rd_kafka_subscribe(rk, topics))){      fprintf(stderr, "%% Failed to start consuming topics: %s\n", rd_kafka_err2str(err));      return -1;  }  return 1;}int main(int argc, char **argv){  char *brokers = "localhost:9092";  char *group = NULL;  char *topic = NULL;    int opt;  rd_kafka_resp_err_t err;  while ((opt = getopt(argc, argv, "g:b:t:qd:eX:As:DO")) != -1){    switch (opt) {      case 'b':        brokers = optarg;        break;      case 'g':        group = optarg;        break;      case 't':        topic = optarg;        break;      default:        break;    }  }   signal(SIGINT, stop);  signal(SIGUSR1, sig_usr1);  if(!initKafka(brokers, group, topic)){    fprintf(stderr, "kafka server initialize error\n");  }else{    while(run){      rd_kafka_message_t *rkmessage;      /*-轮询消费者的消息或事件，最多阻塞timeout_ms        -应用程序应该定期调用consumer_poll()，即使没有预期的消息，以服务        所有排队等待的回调函数，当注册过rebalance_cb，该操作尤为重要，        因为它需要被正确地调用和处理以同步内部消费者状态 */      rkmessage = rd_kafka_consumer_poll(rk, 1000);      if(rkmessage){        msg_consume(rkmessage, NULL);        /*释放rkmessage的资源，并把所有权还给rdkafka*/        rd_kafka_message_destroy(rkmessage);      }    }  }done:    /*此调用将会阻塞，直到consumer撤销其分配，调用rebalance_cb(如果已设置)，    commit offset到broker,并离开consumer group    最大阻塞时间被设置为session.timeout.ms    */    err = rd_kafka_consumer_close(rk);    if(err){      fprintf(stderr, "%% Failed to close consumer: %s\n", rd_kafka_err2str(err));    }else{      fprintf(stderr, "%% Consumer closed\n");    }    //释放topics list使用的所有资源和它自己    rd_kafka_topic_partition_list_destroy(topics);    //destroy kafka handle    rd_kafka_destroy(rk);      run = 5;    //等待所有rd_kafka_t对象销毁，所有kafka对象被销毁，返回0，超时返回-1    while(run-- > 0 && rd_kafka_wait_destroyed(1000) == -1){      printf("Waiting for librdkafka to decommission\n");    }    if(run <= 0){      //dump rdkafka内部状态到stdout流      rd_kafka_dump(stdout, rk);    }    return 0;}

在linux下编译producer和consumer的代码：

gcc my_producer.c -o my_producer  -lrdkafka -lz -lpthread -lrtgcc my_consumer.c -o my_consumer  -lrdkafka -lz -lpthread -lrt

在运行my_producer或my_consumer时可能会报错"error while loading shared libraries xxx.so", 此时需要在/etc/ld.so.conf中加入xxx.so所在的目录

在本地启动一个简单的kafka服务，设置broker集群为localhost：9092并创建一个叫“test_topic”的topic
启动方式可参考 kafka0.8.2集群的环境搭建并实现基本的生产消费

启动consumer：

启动producer，并发送一条数据“hello world”：

consumer处成功收到producer发送的“hello world”：