flume-ng配置文件详解（二）

来源：互联网发布：伊朗空军实力知乎编辑：程序博客网时间：2024/05/17 21:40

4 Syslogtcp案例

[root@localhost apache-flume-1.7.0-bin]# vi conf/tcp.conf
agent1.sources = r1
agent1.sinks = k1
agent1.channels = c1

agent1.sources.r1.type = syslogtcp
agent1.sources.r1.port = 5000
agent1.sources.r1.host = 192.168.100.200
agent1.sources.r1.channels = c1

agent1.sinks.k1.type = logger

agent1.channels.c1.type = memory
agent1.channels.c1.capacity = 1000
agent1.channels.c1.transactionCapacity = 100

agent1.sources.r1.channels = c1
agent1.sinks.k1.channel = c1
[root@localhost apache-flume-1.7.0-bin]# bin/flume-ng agent --conf conf --conf-file conf/tcp.conf --name agent1 -Dfilume.root.loggger=INFO,console

[root@localhost logs]# echo "hello flume" | nc 192.168.100.200 5000
[root@localhost apache-flume-1.7.0-bin]# vi logs/flume.log
21 Nov 2016 00:43:22,220 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.sink.LoggerSink.process:95)

- Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 68 65 6C 6C 6F 20 66 6C 75 6D 65 hello flume }

5 JSONHandler案例

[root@localhost apache-flume-1.7.0-bin]# vi conf/json.conf
agent1.sources = r1
agent1.sinks = k1
agent1.channels = c1

agent1.sources.r1.type = org.apache.flume.source.http.HTTPSource
agent1.sources.r1.port = 8888
agent1.sources.r1.channels = c1

agent1.sinks.k1.type = logger

agent1.channels.c1.type = memory
agent1.channels.c1.capacity = 1000
agent1.channels.c1.transactionCapacity = 100

agent1.sources.r1.channels = c1
agent1.sinks.k1.channel = c1
[root@localhost apache-flume-1.7.0-bin]# bin/flume-ng agent --conf conf --conf-file conf/json.conf --name agent1 -Dfilume.root.loggger=INFO,console
[root@localhost apache-flume-1.7.0-bin]# curl -X POST -d '[{ "headers" :{"a" : "a1","b" : "b1"},"body" : "hello flume"}]' http://192.168.100.200:8888
[root@localhost apache-flume-1.7.0-bin]# vi logs/flume.log
21 Nov 2016 00:50:44,153 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.sink.LoggerSink.process:95)
- Event: { headers:{b=b1, a=a1} body: 68 65 6C 6C 6F 20 66 6C 75 6D 65 hello flume }

6 Hadoop sink案例

[root@localhost apache-flume-1.7.0-bin]# vi conf/hdfs.conf
agent1.sources = r1
agent1.sinks = k1
agent1.channels = c1

agent1.sources.r1.type = syslogtcp
agent1.sources.r1.port = 5140
agent1.sources.r1.host = 192.168.100.200
agent1.sources.r1.channels = c1

agent1.sinks.k1.type = hdfs
agent1.sinks.k1.channel = c1
agent1.sinks.k1.hdfs.path = hdfs://192.168.100.200:9000/flume
agent1.sinks.k1.hdfs.filePrefix = Syslog
agent1.sinks.k1.hdfs.round = true
agent1.sinks.k1.hdfs.roundValue = 10
agent1.sinks.k1.hdfs.roundUnit = minute

agent1.channels.c1.type = memory
agent1.channels.c1.capacity = 1000
agent1.channels.c1.transactionCapacity = 100

agent1.sources.r1.channels = c1
agent1.sinks.k1.channel = c1
[root@localhost apache-flume-1.7.0-bin]# bin/flume-ng agent --conf conf --conf-file conf/hdfs.conf --name agent1 -Dfilume.root.loggger=INFO,console
[root@localhost apache-flume-1.7.0-bin]# echo "hello flume -> hadoop hdfs" | nc 192.168.100.200 5140
pig客户端查看目录
grunt> ls /

7 File Roll Sink案例

[root@localhost apache-flume-1.7.0-bin]# vi conf/fileroll.conf
agent1.sources = r1
agent1.sinks = k1
agent1.channels = c1

agent1.sources.r1.type = syslogtcp
agent1.sources.r1.port = 5555
agent1.sources.r1.host = 192.168.100.200
agent1.sources.r1.channels = c1

agent1.sinks.k1.type = file_roll
agent1.sinks.k1.sink.directory = /usr/local/logs

agent1.channels.c1.type = memory
agent1.channels.c1.capacity = 1000
agent1.channels.c1.transactionCapacity = 100

agent1.sources.r1.channels = c1
agent1.sinks.k1.channel = c1
[root@localhost apache-flume-1.7.0-bin]# echo "hello flume fileroll" | nc 192.168.100.200 5555
[root@localhost apache-flume-1.7.0-bin]# cd /usr/local/logs/
查看具体文件
[root@localhost logs]# cat 1479719885810-2
hello flume fileroll

0 0