flume 学习四：各种执行案例二

1.1     将日志写到hdfs上：a4.conf的内容

#定义agent名， source、channel、sink的名称

a4.sources = r1

a4.channels = c1

a4.sinks = k1

 

#具体定义source

a4.sources.r1.type = spooldir

a4.sources.r1.spoolDir = /home/jurf/flumelog

 

#具体定义channel

a4.channels.c1.type = memory

a4.channels.c1.capacity =10000

a4.channels.c1.transactionCapacity= 100

 

#定义拦截器，为消息添加时间戳

a4.sources.r1.interceptors = i1

a4.sources.r1.interceptors.i1.type= org.apache.flume.interceptor.TimestampInterceptor$Builder

 

 

#具体定义sink

a4.sinks.k1.type = hdfs

a4.sinks.k1.hdfs.path = hdfs://ljf-centos:9000/flume/%Y%m%d     #flume这个目录会自动创建

a4.sinks.k1.hdfs.filePrefix =events-

a4.sinks.k1.hdfs.fileType =DataStream

#不按照条数生成文件

a4.sinks.k1.hdfs.rollCount = 0

#HDFS上的文件达到128M时生成一个文件

a4.sinks.k1.hdfs.rollSize =134217728

#HDFS上的文件达到60秒生成一个文件

a4.sinks.k1.hdfs.rollInterval =60

 

#组装source、channel、sink

a4.sources.r1.channels = c1

a4.sinks.k1.channel = c1

1.2     在/home/jurf/flumelog新建flumelog目录：

    [root@ljf-centos flumelog]# pwd

/home/jurf/flumelog

[root@ljf-centosflumelog]# ls

 [root@ljf-centos flumelog]#

1.3     将日志写到hdfs上，执行结果：

[root@ljf-centosapache-flume-1.6.0-bin]# bin/flume-ng agent -c./conf -f ./conf/a4.conf -n a4 -Dflume.root.logger=INFO,console

1.4     将flumelog2.txt日志文件放入/home/jurf/flumelog下：会变如下格式：

flumelog2.txt.COMPLETED

1.5     显示结果：（注意刚开始size为0，记得刷新）

二.   读取文件的最新日志写入到指定文件中

1.1     读取文件的最新日志写入到指定文件中：example5.conf的内容：

a1.sources = r1

a1.sinks = k1

a1.channels = c1

# Describe/configure the source

a1.sources.r1.type = exec

a1.sources.r1.command =tail -F./mydemo/mylog.log

a1.sources.r1.bind=localhost

a1.sources.r1.port=44444

# Describe the sink

a1.sinks.k1.type =file_roll

a1.sinks.k1.sink.directory=./mydemo/test/

a1.sinks.k1.sink.rollInterval=0

# Use a channel which buffers events inmemory

a1.channels.c1.type = memory

a1.channels.c1.capacity = 10000

a1.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel

a1.sources.r1.channels = c1

a1.sinks.k1.channel = c1

2.执行结果：

打开第一个窗口：

[root@ljf-centos apache-flume-1.6.0-bin]# bin/flume-ng agent -c ./conf -f./conf/example5.conf -n a1 -Dflume.root.logger=INFO,console

打开第二个窗口：向mylog.log这个文件中写入内容

打开第三个窗口：查看

[root@ljf-centos test]# ls

1482934654994-1

[root@ljf-centos test]#

[root@ljf-centos test]# more1482934654994-1

3sdfds

sdfdsfds

sdfdsfds

dsfdsfwerew

ewrewrew

ewrewrew

ewrewrew

ewrewrew

ewrewsd

tyty

mb woshi123

mb woshi345

 

详细文档见百度网盘