python kafka producer consumer redis数据从kafka发送写人redis

来源：互联网发布：nodejs调用java api 编辑：程序博客网时间：2024/05/19 16:05

本例子介绍 python kafka producer发送数据consumer 接收，并且写入redis简单步骤

kafka-python-producer.py

producer 来发送数据，更新时间戳，ID自增长。5秒发送一次

#!/usr/bin/python# -*- coding:utf-8 -*-#kafka-python-producerfrom kafka import KafkaProducerfrom kafka.errors import KafkaErrorimport jsonimport timeimport datetimeproducer = KafkaProducer(bootstrap_servers=['localhost:9092'])HEAD_ID='WP'ID=2020016300386while True:    date=datetime.datetime.strftime(datetime.datetime.now(),"%Y-%m-%d %H:%M:%S")    d={"relateOrder":"1169141000","subSectionCode":"","dealEmpId":"","eventCase":"","isService":"否","eventObject":"物流公司","omsOrderItemId":"00024877154601","eventSource":"LMA","eventCode":"02","isComplain":"否","eventOrder":HEAD_ID+str(ID),"sectionCode":"","isEffectCust":"是","zacty":"A004","eventStatus":"","appearTime":date,"dealDesc":"库存在中转区"}    value=json.dumps(d).encode(encoding="utf-8")    producer.send('test', key=b'123', value=value)    time.sleep(5)    ID+=1    print(value)        kafka-python-consumer.py 接收数据，并存入redis#!/usr/bin/python# -*- coding:utf-8 -*-#kafka-python-consumer.pyfrom kafka import KafkaConsumerimport jsonimport redis# redispool=redis.ConnectionPool(host='localhost',port=6379,db=0)r=redis.Redis(connection_pool=pool)def dict2redis(d):    r.hset(d['eventOrder'],'eventOrder',d['eventOrder'])    r.hset(d['eventOrder'],'relateOrder',d['relateOrder'])    r.hset(d['eventOrder'],'omsOrderItemId',d['omsOrderItemId'])    r.hset(d['eventOrder'],'zacty',d['zacty'])    r.hset(d['eventOrder'],'appearTime',d['appearTime'])    r.hset(d['eventOrder'],'eventSource',d['eventSource'])    #写入redis    # f=open('consumer.txt','w')consumer = KafkaConsumer('test',group_id='my-group',bootstrap_servers=['localhost:9092'])for msg in consumer:        dict=json.loads(msg.value.decode('utf-8'))    print(dict)    dict2redis(dict)print(r.hgetall('WP2020016300386'))

1 0