使用pykafka读取实时数据小例子

来源：互联网发布：数据查找网站编辑：程序博客网时间：2024/06/12 22:36

使用pykafka读取实时数据

import sysfrom pykafka import KafkaClientfrom pykafka.balancedconsumer import BalancedConsumerfrom pykafka.simpleconsumer import OwnedPartition, OffsetTypereload(sys)sys.setdefaultencoding('utf8')#pykafka, need install PyKafkaclass PyKafka:consumer = NoneTOPIC   = 'log_download'BROKER_LIST = '10.23.23.24:9092,10.23.23.21:9092'ZK_LIST = '10.23.23.24:2181,10.23.23.21:2181/sh-bt'server = topic = zsServer = Nonedef __init__(self):print("begin pykafka")self.server  = self.BROKER_LISTself.topic   = self.TOPICself.zkServer= self.ZK_LISTdef getConnect(self):client = KafkaClient(hosts=self.server)topic = client.topics[self.topic]self.consumer = topic.get_balanced_consumer(consumer_group="zs_download_04", # 自己命令auto_offset_reset=OffsetType.LATEST,#在consumer_group存在的情况下，设置此变量，表示从最新的开始取#auto_offset_reset=OffsetType.EARLIEST,#reset_offset_on_start=True,    #auto_commit_enable=True,    zookeeper_connect=self.zkServer)#self.consumer = topic.get_simple_consumer(reset_offset_on_start=False)self.consumer.consume()self.consumer.commit_offsets()return self.consumerdef disConnect(self):#self.consumer.close()passdef beginConsumer(self):for oneLog in self.consumer:print(oneLog.offset)print(oneLog.value)if __name__ == '__main__':pk = PyKafka()pk.getConnect()pk.beginConsumer()

0 0