flume实时接收kafka消息并保存至HDFS

来源：互联网发布：公务员考试经验知乎编辑：程序博客网时间：2024/06/05 17:36

#-------agent------

# 定义sources
agent.sources = kafkaSource

# 定义channels
agent.channels = memoryChannel

# 定义sinks
agent.sinks = hdfsSink
agent.sources.kafkaSource.channels = memoryChannel
agent.sinks.hdfsSink.channel = memoryChannel

#-------- kafkaSource-----------------
agent.sources.kafkaSource.type = org.apache.flume.source.kafka.KafkaSource
agent.sources.kafkaSource.zookeeperConnect = h2:2181,h3:2181,h4:2181,h8:2181,h9:2181,h10:2181
agent.sources.kafkaSource.topic = T20161031
#agent.sources.kafkaSource.groupId = Group_20161031
agent.sources.kafkaSource.kafka.consumer.timeout.ms = 1000

#------- memoryChannel-------------------------
agent.channels.memoryChannel.type = memory
agent.channels.memoryChannel.capacity=10000
agent.channels.memoryChannel.transactionCapacity=1000

#---------hdfsSink ------------------
agent.sinks.hdfsSink.type = hdfs
agent.sinks.hdfsSink.hdfs.path = hdfs://h4:9000/user/test/kafka2HdfsByFlume
agent.sinks.hdfsSink.hdfs.writeFormat = Text
agent.sinks.hdfsSink.hdfs.fileType = DataStream

0 0