程序博客网 > 折八百淘宝商城马甲

Flume安装及简单使用

来源：互联网 发布：折八百淘宝商城马甲 编辑：程序博客网 时间：2024/06/05 11:00

1下载解压到指定目录
22)修改 flume-env.sh 配置文件,主要是JAVA_HOME变量设置

1.root@m1:/home/hadoop/flume-1.5.0-bin# cp conf/flume-env.sh.template conf/flume-env.sh
2.root@m1:/home/hadoop/flume-1.5.0-bin# vi conf/flume-env.sh
3.# Licensed to the Apache Software Foundation (ASF) under one
4.# or more contributor license agreements. See the NOTICE file
5.# distributed with this work for additional information
6.# regarding copyright ownership. The ASF licenses this file
7.# to you under the Apache License, Version 2.0 (the
8.# “License”); you may not use this file except in compliance
9.# with the License. You may obtain a copy of the License at
10.#
11.# http://www.apache.org/licenses/LICENSE-2.0
12.#
13.# Unless required by applicable law or agreed to in writing, software
14.# distributed under the License is distributed on an “AS IS” BASIS,
15.# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16.# See the License for the specific language governing permissions and
17.# limitations under the License.
18.
19.# If this file is placed at FLUME_CONF_DIR/flume-env.sh, it will be sourced
20.# during Flume startup.
21.
22.# Enviroment variables can be set here.
23.
24.JAVA_HOME=/usr/lib/jvm/java-7-oracle
25.
26.# Give Flume more memory and pre-allocate, enable remote monitoring via JMX
27.#JAVA_OPTS=”-Xms100m -Xmx200m -Dcom.sun.management.jmxremote”
28.
29.# Note that the Flume conf directory is always included in the classpath.
30.#FLUME_CLASSPATH=””
复制代码

          3)验证是否安装成功

1.root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng version
2.Flume 1.5.0
3.Source code repository: https://git-wip-us.apache.org/repos/asf/flume.git
4.Revision: 8633220df808c4cd0c13d1cf0320454a94f1ea97
5.Compiled by hshreedharan on Wed May 7 14:49:18 PDT 2014
6.From source with checksum a01fe726e4380ba0c9f7a7d222db961f
7.root@m1:/home/hadoop#

复制代码
　　　　出现上面的信息，表示安装成功了

　　五、flume的案例
　　　　1)案例1：Avro
　　　　Avro可以发送一个给定的文件给Flume，Avro 源使用AVRO RPC机制。
　　　　　　a)创建agent配置文件

  1. root@m1:/home/hadoop#vi /home/hadoop/flume-1.5.0-bin/conf/avro.conf

    1. a1.sources = r1
  3. a1.sinks = k1
  4. a1.channels = c1

    1. # Describe/configure the source
  6. a1.sources.r1.type = avro
  7. a1.sources.r1.channels = c1
  8. a1.sources.r1.bind = 0.0.0.0
  9. a1.sources.r1.port = 4141

    1. # Describe the sink
  11. a1.sinks.k1.type = logger

    1. # Use a channel which buffers events in memory
  13. a1.channels.c1.type = memory
  14. a1.channels.c1.capacity = 1000
  15. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  17. a1.sources.r1.channels = c1

  18. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　b)启动flume agent a1

  19. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/avro.conf -n a1 -Dflume.root.logger=INFO,console

  20. 复制代码

    　　　　　　c)创建指定文件

  21. root@m1:/home/hadoop# echo “hello world” > /home/hadoop/flume-1.5.0-bin/log.00
    复制代码

    　　　　　　d)使用avro-client发送文件

  22. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng avro-client -c . -H m1 -p 4141 -F /home/hadoop/flume-1.5.0-bin/log.00
    复制代码

    　　　　　　d)使用avro-client发送文件

  23. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng avro-client -c . -H m1 -p 4141 -F /
    复制代码

    　　　　　　f)在m1的控制台，可以看到以下信息，注意最后一行：

  24. root@m1:/home/hadoop/flume-1.5.0-bin/conf# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/avro.conf -n a1 -Dflume.root.logger=INFO,console

  25. Info: Sourcing environment configuration script /home/hadoop/flume-1.5.0-bin/conf/flume-env.sh
  26. Info: Including Hadoop libraries found via (/home/hadoop/hadoop-2.2.0/bin/hadoop) for HDFS access
  27. Info: Excluding /home/hadoop/hadoop-2.2.0/share/hadoop/common/lib/slf4j-api-1.7.5.jar from classpath
  28. Info: Excluding /home/hadoop/hadoop-2.2.0/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar from classpath
  29. …
  30. 2014-08-10 10:43:25,112 (New I/O worker #1) [INFO - org.apache.avro.ipc.NettyServer$NettyServerAvroHandler.handleUpstream(NettyServer.java:171)] [id: 0x92464c4f, /192.168.1.50:59850 :> /192.168.1.50:4141] UNBOUND
  31. 2014-08-10 10:43:25,112 (New I/O worker #1) [INFO - org.apache.avro.ipc.NettyServer$NettyServerAvroHandler.handleUpstream(NettyServer.java:171)] [id: 0x92464c4f, /192.168.1.50:59850 :> /192.168.1.50:4141] CLOSED
  32. 2014-08-10 10:43:25,112 (New I/O worker #1) [INFO - org.apache.avro.ipc.NettyServer$NettyServerAvroHandler.channelClosed(NettyServer.java:209)] Connection to /192.168.1.50:59850 disconnected.
  33. 2014-08-10 10:43:26,718 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 68 65 6C 6C 6F 20 77 6F 72 6C 64 hello world }

  34. 复制代码

    　　　　2)案例2：Spool
    　　　　Spool监测配置的目录下新增的文件，并将文件中的数据读取出来。需要注意两点：
    　　　　1) 拷贝到spool目录下的文件不可以再打开编辑。
    　　　　2) spool目录下不可包含相应的子目录

    　　　　　　a)创建agent配置文件

  35. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/spool.conf


    1. a1.sources = r1
  37. a1.sinks = k1
  38. a1.channels = c1

    1. # Describe/configure the source
  40. a1.sources.r1.type = spooldir
  41. a1.sources.r1.channels = c1
  42. a1.sources.r1.spoolDir = /home/hadoop/flume-1.5.0-bin/logs
  43. a1.sources.r1.fileHeader = true

    1. # Describe the sink
  45. a1.sinks.k1.type = logger

    1. # Use a channel which buffers events in memory
  47. a1.channels.c1.type = memory
  48. a1.channels.c1.capacity = 1000
  49. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  51. a1.sources.r1.channels = c1
  52. a1.sinks.k1.channel = c1

  53. 复制代码

    　　　　　　b)启动flume agent a1

  54. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/spool.conf -n a1 -Dflume.root.logger=INFO,console

  55. 复制代码

    　　　　　　c)追加文件到/home/hadoop/flume-1.5.0-bin/logs目录

  56. root@m1:/home/hadoop# echo “spool test1” > /home/hadoop/flume-1.5.0-bin/logs/spool_text.log

  57. 复制代码

    　　　　　d)在m1的控制台，可以看到以下相关信息：

  58. 14/08/10 11:37:13 INFO source.SpoolDirectorySource: Spooling Directory Source runner has shutdown.

  59. 14/08/10 11:37:13 INFO source.SpoolDirectorySource: Spooling Directory Source runner has shutdown.
  60. 14/08/10 11:37:14 INFO avro.ReliableSpoolingFileEventReader: Preparing to move file /home/hadoop/flume-1.5.0-bin/logs/spool_text.log to /home/hadoop/flume-1.5.0-bin/logs/spool_text.log.COMPLETED
  61. 14/08/10 11:37:14 INFO source.SpoolDirectorySource: Spooling Directory Source runner has shutdown.
  62. 14/08/10 11:37:14 INFO source.SpoolDirectorySource: Spooling Directory Source runner has shutdown.
  63. 14/08/10 11:37:14 INFO sink.LoggerSink: Event: { headers:{file=/home/hadoop/flume-1.5.0-bin/logs/spool_text.log} body: 73 70 6F 6F 6C 20 74 65 73 74 31 spool test1 }
  64. 14/08/10 11:37:15 INFO source.SpoolDirectorySource: Spooling Directory Source runner has shutdown.
  65. 14/08/10 11:37:15 INFO source.SpoolDirectorySource: Spooling Directory Source runner has shutdown.
  66. 14/08/10 11:37:16 INFO source.SpoolDirectorySource: Spooling Directory Source runner has shutdown.
  67. 14/08/10 11:37:16 INFO source.SpoolDirectorySource: Spooling Directory Source runner has shutdown.

  68. 14/08/10 11:37:17 INFO source.SpoolDirectorySource: Spooling Directory Source runner has shutdown.
    复制代码

    　　　　3)案例3：Exec
    　　　　EXEC执行一个给定的命令获得输出的源,如果要使用tail命令，必选使得file足够大才能看到输出内容

    　　　　　　a)创建agent配置文件

  69. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/exec_tail.conf


    1. a1.sources = r1
  71. a1.sinks = k1
  72. a1.channels = c1

    1. # Describe/configure the source
  74. a1.sources.r1.type = exec
  75. a1.sources.r1.channels = c1
  76. a1.sources.r1.command = tail -F /home/hadoop/flume-1.5.0-bin/log_exec_tail

    1. # Describe the sink
  78. a1.sinks.k1.type = logger

    1. # Use a channel which buffers events in memory
  80. a1.channels.c1.type = memory
  81. a1.channels.c1.capacity = 1000
  82. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  84. a1.sources.r1.channels = c1

  85. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　b)启动flume agent a1

  86. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/exec_tail.conf -n a1 -Dflume.root.logger=INFO,console

  87. 复制代码

    　　　　　　c)生成足够多的内容在文件里

  88. root@m1:/home/hadoop# for i in {1..100};do echo “exec tail$i” >> /home/hadoop/flume-1.5.0-bin/log_
    复制代码

    　　　　　　e)在m1的控制台，可以看到以下信息：

  89. 2014-08-10 10:59:25,513 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 20 74 65 73 74 exec tail test }

  90. 2014-08-10 10:59:34,535 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 20 74 65 73 74 exec tail test }
  91. 2014-08-10 11:01:40,557 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 31 exec tail1 }
  92. 2014-08-10 11:01:41,180 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 32 exec tail2 }
  93. 2014-08-10 11:01:41,180 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 33 exec tail3 }
  94. 2014-08-10 11:01:41,181 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 34 exec tail4 }
  95. 2014-08-10 11:01:41,181 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 35 exec tail5 }
  96. 2014-08-10 11:01:41,181 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 36 exec tail6 }
  97. ….
  98. ….
  99. ….
  100. 2014-08-10 11:01:51,550 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 39 36 exec tail96 }
  101. 2014-08-10 11:01:51,550 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 39 37 exec tail97 }
  102. 2014-08-10 11:01:51,551 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 39 38 exec tail98 }
  103. 2014-08-10 11:01:51,551 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 39 39 exec tail99 }
  104. 2014-08-10 11:01:51,551 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:70)] Event: { headers:{} body: 65 78 65 63 20 74 61 69 6C 31 30 30 exec tail100 }

  105. 复制代码

    　　　　4)案例4：Syslogtcp
    　　　　Syslogtcp监听TCP的端口做为数据源

    　　　　　　a)创建agent配置文件

  106. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/syslog_tcp.conf


    1. a1.sources = r1
  108. a1.sinks = k1
  109. a1.channels = c1

    1. # Describe/configure the source
  111. a1.sources.r1.type = syslogtcp
  112. a1.sources.r1.port = 5140
  113. a1.sources.r1.host = localhost
  114. a1.sources.r1.channels = c1

    1. # Describe the sink
  116. a1.sinks.k1.type = logger

    1. # Use a channel which buffers events in memory
  118. a1.channels.c1.type = memory
  119. a1.channels.c1.capacity = 1000
  120. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  122. a1.sources.r1.channels = c1

  123. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　b)启动flume agent a1

  124. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/syslog_tcp.conf -n a1 -Dflume.root.logger=INFO,console

  125. 复制代码

    　　　　　　c)测试产生syslog

  126. root@m1:/home/hadoop# echo “hello idoall.org syslog” | nc localhost 5140
    复制代码

    　　　　　　d)在m1的控制台，可以看到以下信息：

  127. 14/08/10 11:41:45 INFO node.PollingPropertiesFileConfigurationProvider: Reloading configuration file:/home/hadoop/flume-1.5.0-bin/conf/syslog_tcp.conf

  128. 14/08/10 11:41:45 INFO conf.FlumeConfiguration: Added sinks: k1 Agent: a1
  129. 14/08/10 11:41:45 INFO conf.FlumeConfiguration: Processing:k1
  130. 14/08/10 11:41:45 INFO conf.FlumeConfiguration: Processing:k1
  131. 14/08/10 11:41:45 INFO conf.FlumeConfiguration: Post-validation flume configuration contains configuration for agents: [a1]
  132. 14/08/10 11:41:45 INFO node.AbstractConfigurationProvider: Creating channels
  133. 14/08/10 11:41:45 INFO channel.DefaultChannelFactory: Creating instance of channel c1 type memory
  134. 14/08/10 11:41:45 INFO node.AbstractConfigurationProvider: Created channel c1
  135. 14/08/10 11:41:45 INFO source.DefaultSourceFactory: Creating instance of source r1, type syslogtcp
  136. 14/08/10 11:41:45 INFO sink.DefaultSinkFactory: Creating instance of sink: k1, type: logger
  137. 14/08/10 11:41:45 INFO node.AbstractConfigurationProvider: Channel c1 connected to [r1, k1]
  138. 14/08/10 11:41:45 INFO node.Application: Starting new configuration:{ sourceRunners:{r1=EventDrivenSourceRunner: { source:org.apache.flume.source.SyslogTcpSource{name:r1,state:IDLE} }} sinkRunners:{k1=SinkRunner: { policy:org.apache.flume.sink.DefaultSinkProcessor@6538b14 counterGroup:{ name:null counters:{} } }} channels:{c1=org.apache.flume.channel.MemoryChannel{name: c1}} }
  139. 14/08/10 11:41:45 INFO node.Application: Starting Channel c1
  140. 14/08/10 11:41:45 INFO instrumentation.MonitoredCounterGroup: Monitored counter group for type: CHANNEL, name: c1: Successfully registered new MBean.
  141. 14/08/10 11:41:45 INFO instrumentation.MonitoredCounterGroup: Component type: CHANNEL, name: c1 started
  142. 14/08/10 11:41:45 INFO node.Application: Starting Sink k1
  143. 14/08/10 11:41:45 INFO node.Application: Starting Source r1
  144. 14/08/10 11:41:45 INFO source.SyslogTcpSource: Syslog TCP Source starting…
  145. 14/08/10 11:42:15 WARN source.SyslogUtils: Event created from Invalid Syslog data.

  146. 14/08/10 11:42:15 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 68 65 6C 6C 6F 20 69 64 6F 61 6C 6C 2E 6F 72 67 hello idoall.org }
    复制代码

    　　　　5)案例5：JSONHandler
    　　　　　　a)创建agent配置文件

  147. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/post_json.conf


    1. a1.sources = r1
  149. a1.sinks = k1
  150. a1.channels = c1

    1. # Describe/configure the source
  152. a1.sources.r1.type = org.apache.flume.source.http.HTTPSource
  153. a1.sources.r1.port = 8888
  154. a1.sources.r1.channels = c1

    1. # Describe the sink
  156. a1.sinks.k1.type = logger

    1. # Use a channel which buffers events in memory
  158. a1.channels.c1.type = memory
  159. a1.channels.c1.capacity = 1000
  160. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  162. a1.sources.r1.channels = c1

  163. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　b)启动flume agent a1

  164. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/post_json.conf -n a1 -Dflume.root.logger=INFO,console
    复制代码

    　　　　　　c)生成JSON 格式的POST request

  165. root@m1:/home/hadoop# curl -X POST -d ‘[{ “headers” :{“a” : “a1”,”b” : “b1”},”body” : “idoall.org_body”}]’ http://localhost:8888

  166. 复制代码

    　　　　　　d)在m1的控制台，可以看到以下信息：

  167. 14/08/10 11:49:59 INFO node.Application: Starting Channel c1

  168. 14/08/10 11:49:59 INFO instrumentation.MonitoredCounterGroup: Monitored counter group for type: CHANNEL, name: c1: Successfully registered new MBean.
  169. 14/08/10 11:49:59 INFO instrumentation.MonitoredCounterGroup: Component type: CHANNEL, name: c1 started
  170. 14/08/10 11:49:59 INFO node.Application: Starting Sink k1
  171. 14/08/10 11:49:59 INFO node.Application: Starting Source r1
  172. 14/08/10 11:49:59 INFO mortbay.log: Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
  173. 14/08/10 11:49:59 INFO mortbay.log: jetty-6.1.26
  174. 14/08/10 11:50:00 INFO mortbay.log: Started SelectChannelConnector@0.0.0.0:8888
  175. 14/08/10 11:50:00 INFO instrumentation.MonitoredCounterGroup: Monitored counter group for type: SOURCE, name: r1: Successfully registered new MBean.
  176. 14/08/10 11:50:00 INFO instrumentation.MonitoredCounterGroup: Component type: SOURCE, name: r1 started
  177. 14/08/10 12:14:32 INFO sink.LoggerSink: Event: { headers:{b=b1, a=a1} body: 69 64 6F 61 6C 6C 2E 6F 72 67 5F 62 6F 64 79 idoall.org_body }

  178. 复制代码

    　　　　6)案例6：Hadoop sink
    　　　　　　a)创建agent配置文件

  179. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/hdfs_sink.conf


    1. a1.sources = r1
  181. a1.sinks = k1
  182. a1.channels = c1

    1. # Describe/configure the source
  184. a1.sources.r1.type = syslogtcp
  185. a1.sources.r1.port = 5140
  186. a1.sources.r1.host = localhost
  187. a1.sources.r1.channels = c1

    1. # Describe the sink
  189. a1.sinks.k1.type = hdfs
  190. a1.sinks.k1.channel = c1
  191. a1.sinks.k1.hdfs.path = hdfs://m1:9000/user/flume/syslogtcp
  192. a1.sinks.k1.hdfs.filePrefix = Syslog
  193. a1.sinks.k1.hdfs.round = true
  194. a1.sinks.k1.hdfs.roundValue = 10
  195. a1.sinks.k1.hdfs.roundUnit = minute

    1. # Use a channel which buffers events in memory
  197. a1.channels.c1.type = memory
  198. a1.channels.c1.capacity = 1000
  199. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  201. a1.sources.r1.channels = c1

  202. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　b)启动flume agent a1

  203. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/hdfs_sink.conf -n a1 -Dflume.root.logger=INFO,console
    复制代码

    　　　　　　c)测试产生syslog

  204. root@m1:/home/hadoop# echo “hello idoall flume -> hadoop testing one” | nc localhost 5140

  205. 复制代码

    　　　　　　d)在m1的控制台，可以看到以下信息：

  206. 14/08/10 12:20:39 INFO instrumentation.MonitoredCounterGroup: Monitored counter group for type: CHANNEL, name: c1: Successfully registered new MBean.

  207. 14/08/10 12:20:39 INFO instrumentation.MonitoredCounterGroup: Component type: CHANNEL, name: c1 started
  208. 14/08/10 12:20:39 INFO node.Application: Starting Sink k1
  209. 14/08/10 12:20:39 INFO node.Application: Starting Source r1
  210. 14/08/10 12:20:39 INFO instrumentation.MonitoredCounterGroup: Monitored counter group for type: SINK, name: k1: Successfully registered new MBean.
  211. 14/08/10 12:20:39 INFO instrumentation.MonitoredCounterGroup: Component type: SINK, name: k1 started
  212. 14/08/10 12:20:39 INFO source.SyslogTcpSource: Syslog TCP Source starting…
  213. 14/08/10 12:21:46 WARN source.SyslogUtils: Event created from Invalid Syslog data.
  214. 14/08/10 12:21:49 INFO hdfs.HDFSSequenceFile: writeFormat = Writable, UseRawLocalFileSystem = false
  215. 14/08/10 12:21:49 INFO hdfs.BucketWriter: Creating hdfs://m1:9000/user/flume/syslogtcp//Syslog.1407644509504.tmp
  216. 14/08/10 12:22:20 INFO hdfs.BucketWriter: Closing hdfs://m1:9000/user/flume/syslogtcp//Syslog.1407644509504.tmp
  217. 14/08/10 12:22:20 INFO hdfs.BucketWriter: Close tries incremented
  218. 14/08/10 12:22:20 INFO hdfs.BucketWriter: Renaming hdfs://m1:9000/user/flume/syslogtcp/Syslog.1407644509504.tmp to hdfs://m1:9000/user/flume/syslogtcp/Syslog.1407644509504

  219. 14/08/10 12:22:20 INFO hdfs.HDFSEventSink: Writer callback called.
    复制代码

    　　　　　　e)在m1上再打开一个窗口，去hadoop上检查文件是否生成

  220. root@m1:/home/hadoop# /home/hadoop/hadoop-2.2.0/bin/hadoop fs -ls /user/flume/syslogtcp

  221. Found 1 items
  222. -rw-r–r– 3 root supergroup 155 2014-08-10 12:22 /user/flume/syslogtcp/Syslog.1407644509504
  223. root@m1:/home/hadoop# /home/hadoop/hadoop-2.2.0/bin/hadoop fs -cat /user/flume/syslogtcp/Syslog.1407644509504
  224. SEQ!org.apache.hadoop.io.LongWritable”org.apache.hadoop.io.BytesWritable^;>Gv$hello idoall flume -> hadoop testing one

  225. 复制代码

    　　　　7)案例7：File Roll Sink
    　　　　　　a)创建agent配置文件

  226. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/file_roll.conf


    1. a1.sources = r1
  228. a1.sinks = k1
  229. a1.channels = c1

    1. # Describe/configure the source
  231. a1.sources.r1.type = syslogtcp
  232. a1.sources.r1.port = 5555
  233. a1.sources.r1.host = localhost
  234. a1.sources.r1.channels = c1

    1. # Describe the sink
  236. a1.sinks.k1.type = file_roll
  237. a1.sinks.k1.sink.directory = /home/hadoop/flume-1.5.0-bin/logs

    1. # Use a channel which buffers events in memory
  239. a1.channels.c1.type = memory
  240. a1.channels.c1.capacity = 1000
  241. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  243. a1.sources.r1.channels = c1

  244. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　b)启动flume agent a1

  245. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/file_roll.conf -n a1 -Dflume.root.logger=INFO,console

  246. 复制代码

    　　　　　　c)测试产生log

  247. root@m1:/home/hadoop# echo “hello idoall.org syslog” | nc localhost 5555

  248. root@m1:/home/hadoop# echo “hello idoall.org syslog 2” | nc localhost 5555
    复制代码

    　　　　　　d)查看/home/hadoop/flume-1.5.0-bin/logs下是否生成文件,默认每30秒生成一个新文件

  249. root@m1:/home/hadoop# ll /home/hadoop/flume-1.5.0-bin/logs

  250. 总用量 272
  251. drwxr-xr-x 3 root root 4096 Aug 10 12:50 ./
  252. drwxr-xr-x 9 root root 4096 Aug 10 10:59 ../
  253. -rw-r–r– 1 root root 50 Aug 10 12:49 1407646164782-1
  254. -rw-r–r– 1 root root 0 Aug 10 12:49 1407646164782-2
  255. -rw-r–r– 1 root root 0 Aug 10 12:50 1407646164782-3
  256. root@m1:/home/hadoop# cat /home/hadoop/flume-1.5.0-bin/logs/1407646164782-1 /home/hadoop/flume-1.5.0-bin/logs/1407646164782-2
  257. hello idoall.org syslog

  258. hello idoall.org syslog 2
    复制代码

    　　　　8)案例8：Replicating Channel Selector
    　　　　Flume支持Fan out流从一个源到多个通道。有两种模式的Fan out，分别是复制和复用。在复制的情况下，流的事件被发送到所有的配置通道。在复用的情况下，事件被发送到可用的渠道中的一个子集。Fan out流需要指定源和Fan out通道的规则。

    　　　　这次我们需要用到m1,m2两台机器

    　　　　　　a)在m1创建replicating_Channel_Selector配置文件

  259. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/replicating_Channel_Selector.conf

    1. a1.sources = r1
  261. a1.sinks = k1 k2
  262. a1.channels = c1 c2

    1. # Describe/configure the source
  264. a1.sources.r1.type = syslogtcp
  265. a1.sources.r1.port = 5140
  266. a1.sources.r1.host = localhost
  267. a1.sources.r1.channels = c1 c2
  268. a1.sources.r1.selector.type = replicating

    1. # Describe the sink
  270. a1.sinks.k1.type = avro
  271. a1.sinks.k1.channel = c1
  272. a1.sinks.k1.hostname = m1
  273. a1.sinks.k1.port = 5555

    1. a1.sinks.k2.type = avro
  275. a1.sinks.k2.channel = c2
  276. a1.sinks.k2.hostname = m2
  277. a1.sinks.k2.port = 5555

    1. # Use a channel which buffers events in memory
  279. a1.channels.c1.type = memory
  280. a1.channels.c1.capacity = 1000
  281. a1.channels.c1.transactionCapacity = 100

    1. a1.channels.c2.type = memory
  283. a1.channels.c2.capacity = 1000

  284. a1.channels.c2.transactionCapacity = 100
    复制代码

    　　　　　　b)在m1创建replicating_Channel_Selector_avro配置文件

  285. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/replicating_Channel_Selector_avro.conf

    1. a1.sources = r1
  287. a1.sinks = k1
  288. a1.channels = c1

    1. # Describe/configure the source
  290. a1.sources.r1.type = avro
  291. a1.sources.r1.channels = c1
  292. a1.sources.r1.bind = 0.0.0.0
  293. a1.sources.r1.port = 5555

    1. # Describe the sink
  295. a1.sinks.k1.type = logger

    1. # Use a channel which buffers events in memory
  297. a1.channels.c1.type = memory
  298. a1.channels.c1.capacity = 1000
  299. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  301. a1.sources.r1.channels = c1

  302. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　c)在m1上将2个配置文件复制到m2上一份

  303. root@m1:/home/hadoop/flume-1.5.0-bin# scp -r /home/hadoop/flume-1.5.0-bin/conf/replicating_Channel_Selector.conf root@m2:/home/hadoop/flume-1.5.0-bin/conf/replicating_Channel_Selector.conf
  304. root@m1:/home/hadoop/flume-1.5.0-bin# scp -r /home/hadoop/flume-1.5.0-bin/conf/replicating_Channel_Selector_avro.conf root@m2:/home/hadoop/flume-1.5.0-bin/conf/replicating_Channel_Selector_avro.conf

  305. 复制代码
    　　　　　　d)打开4个窗口，在m1和m2上同时启动两个flume agent
  306. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/replicating_Channel_Selector_avro.conf -n a1 -Dflume.root.logger=INFO,console
  307. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/replicating_Channel_Selector.conf -n a1 -Dflume.root.logger=INFO,console

  308. 复制代码

    　　　　　　e)然后在m1或m2的任意一台机器上，测试产生syslog

  309. root@m1:/home/hadoop# echo “hello idoall.org syslog” | nc localhost 5140

  310. 复制代码

    　　　　　　f)在m1和m2的sink窗口，分别可以看到以下信息,这说明信息得到了同步：

  311. 14/08/10 14:08:18 INFO ipc.NettyServer: Connection to /192.168.1.51:46844 disconnected.
  312. 14/08/10 14:08:52 INFO ipc.NettyServer: [id: 0x90f8fe1f, /192.168.1.50:35873 => /192.168.1.50:5555] OPEN
  313. 14/08/10 14:08:52 INFO ipc.NettyServer: [id: 0x90f8fe1f, /192.168.1.50:35873 => /192.168.1.50:5555] BOUND: /192.168.1.50:5555
  314. 14/08/10 14:08:52 INFO ipc.NettyServer: [id: 0x90f8fe1f, /192.168.1.50:35873 => /192.168.1.50:5555] CONNECTED: /192.168.1.50:35873
  315. 14/08/10 14:08:59 INFO ipc.NettyServer: [id: 0xd6318635, /192.168.1.51:46858 => /192.168.1.50:5555] OPEN
  316. 14/08/10 14:08:59 INFO ipc.NettyServer: [id: 0xd6318635, /192.168.1.51:46858 => /192.168.1.50:5555] BOUND: /192.168.1.50:5555
  317. 14/08/10 14:08:59 INFO ipc.NettyServer: [id: 0xd6318635, /192.168.1.51:46858 => /192.168.1.50:5555] CONNECTED: /192.168.1.51:46858
  318. 14/08/10 14:09:20 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 68 65 6C 6C 6F 20 69 64 6F 61 6C 6C 2E 6F 72 67 hello idoall.org }

  319. 复制代码

    　　　　9)案例9：Multiplexing Channel Selector
    　　　　　　a)在m1创建Multiplexing_Channel_Selector配置文件

  320. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/Multiplexing_Channel_Selector.conf


    1. a1.sources = r1
  322. a1.sinks = k1 k2
  323. a1.channels = c1 c2

    1. # Describe/configure the source
  325. a1.sources.r1.type = org.apache.flume.source.http.HTTPSource
  326. a1.sources.r1.port = 5140
  327. a1.sources.r1.channels = c1 c2
  328. a1.sources.r1.selector.type = multiplexing

    1. a1.sources.r1.selector.header = type
  330. #映射允许每个值通道可以重叠。默认值可以包含任意数量的通道。
  331. a1.sources.r1.selector.mapping.baidu = c1
  332. a1.sources.r1.selector.mapping.ali = c2
  333. a1.sources.r1.selector.default = c1

    1. # Describe the sink
  335. a1.sinks.k1.type = avro
  336. a1.sinks.k1.channel = c1
  337. a1.sinks.k1.hostname = m1
  338. a1.sinks.k1.port = 5555

    1. a1.sinks.k2.type = avro
  340. a1.sinks.k2.channel = c2
  341. a1.sinks.k2.hostname = m2
  342. a1.sinks.k2.port = 5555

    1. # Use a channel which buffers events in memory
  344. a1.channels.c1.type = memory
  345. a1.channels.c1.capacity = 1000
  346. a1.channels.c1.transactionCapacity = 100

    1. a1.channels.c2.type = memory
  348. a1.channels.c2.capacity = 1000

  349. a1.channels.c2.transactionCapacity = 100
    复制代码

    　　　　　　b)在m1创建Multiplexing_Channel_Selector_avro配置文件

  350. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/Multiplexing_Channel_Selector_avro.conf

    1. a1.sources = r1
  352. a1.sinks = k1
  353. a1.channels = c1

    1. # Describe/configure the source
  355. a1.sources.r1.type = avro
  356. a1.sources.r1.channels = c1
  357. a1.sources.r1.bind = 0.0.0.0
  358. a1.sources.r1.port = 5555

    1. # Describe the sink
  360. a1.sinks.k1.type = logger

    1. # Use a channel which buffers events in memory
  362. a1.channels.c1.type = memory
  363. a1.channels.c1.capacity = 1000
  364. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  366. a1.sources.r1.channels = c1

  367. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　c)将2个配置文件复制到m2上一份

  368. root@m1:/home/hadoop/flume-1.5.0-bin# scp -r /home/hadoop/flume-1.5.0-bin/conf/Multiplexing_Channel_Selector.conf root@m2:/home/hadoop/flume-1.5.0-bin/conf/Multiplexing_Channel_Selector.conf
  369. root@m1:/home/hadoop/flume-1.5.0-bin# scp -r /home/hadoop/flume-1.5.0-bin/conf/Multiplexing_Channel_Selector_avro.conf root@m2:/home/hadoop/flume-1.5.0-bin/conf/Multiplexing_Channel_Selector_avro.conf

  370. 复制代码

    　　　　　　d)打开4个窗口，在m1和m2上同时启动两个flume agent

  371. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/Multiplexing_Channel_Selector_avro.conf -n a1 -Dflume.root.logger=INFO,console
  372. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/Multiplexing_Channel_Selector.conf -n a1 -Dflume.root.logger=INFO,console

  373. 复制代码

    　　　　　　e)然后在m1或m2的任意一台机器上，测试产生syslog

  374. root@m1:/home/hadoop# curl -X POST -d ‘[{ “headers” :{“type” : “baidu”},”body” : “idoall_TEST1”}]’ http://localhost:5140 && curl -X POST -d ‘[{ “headers” :{“type” : “ali”},”body” : “idoall_TEST2”}]’ http://localhost:5140 && curl -X POST -d ‘[{ “headers” :{“type” : “qq”},”body” : “idoall_TEST3”}]’ http://localhost:5140

  375. 复制代码

    　　　　　f)在m1的sink窗口，可以看到以下信息：

  376. 14/08/10 14:32:21 INFO node.Application: Starting Sink k1
  377. 14/08/10 14:32:21 INFO node.Application: Starting Source r1
  378. 14/08/10 14:32:21 INFO source.AvroSource: Starting Avro source r1: { bindAddress: 0.0.0.0, port: 5555 }…
  379. 14/08/10 14:32:21 INFO instrumentation.MonitoredCounterGroup: Monitored counter group for type: SOURCE, name: r1: Successfully registered new MBean.
  380. 14/08/10 14:32:21 INFO instrumentation.MonitoredCounterGroup: Component type: SOURCE, name: r1 started
  381. 14/08/10 14:32:21 INFO source.AvroSource: Avro source r1 started.
  382. 14/08/10 14:32:36 INFO ipc.NettyServer: [id: 0xcf00eea6, /192.168.1.50:35916 => /192.168.1.50:5555] OPEN
  383. 14/08/10 14:32:36 INFO ipc.NettyServer: [id: 0xcf00eea6, /192.168.1.50:35916 => /192.168.1.50:5555] BOUND: /192.168.1.50:5555
  384. 14/08/10 14:32:36 INFO ipc.NettyServer: [id: 0xcf00eea6, /192.168.1.50:35916 => /192.168.1.50:5555] CONNECTED: /192.168.1.50:35916
  385. 14/08/10 14:32:44 INFO ipc.NettyServer: [id: 0x432f5468, /192.168.1.51:46945 => /192.168.1.50:5555] OPEN
  386. 14/08/10 14:32:44 INFO ipc.NettyServer: [id: 0x432f5468, /192.168.1.51:46945 => /192.168.1.50:5555] BOUND: /192.168.1.50:5555
  387. 14/08/10 14:32:44 INFO ipc.NettyServer: [id: 0x432f5468, /192.168.1.51:46945 => /192.168.1.50:5555] CONNECTED: /192.168.1.51:46945
  388. 14/08/10 14:34:11 INFO sink.LoggerSink: Event: { headers:{type=baidu} body: 69 64 6F 61 6C 6C 5F 54 45 53 54 31 idoall_TEST1 }
  389. 14/08/10 14:34:57 INFO sink.LoggerSink: Event: { headers:{type=qq} body: 69 64 6F 61 6C 6C 5F 54 45 53 54 33 idoall_TEST3 }

  390. 复制代码

    　　　　　g)在m2的sink窗口，可以看到以下信息：

  391. 14/08/10 14:32:27 INFO node.Application: Starting Sink k1

  392. 14/08/10 14:32:27 INFO node.Application: Starting Source r1
  393. 14/08/10 14:32:27 INFO source.AvroSource: Starting Avro source r1: { bindAddress: 0.0.0.0, port: 5555 }…
  394. 14/08/10 14:32:27 INFO instrumentation.MonitoredCounterGroup: Monitored counter group for type: SOURCE, name: r1: Successfully registered new MBean.
  395. 14/08/10 14:32:27 INFO instrumentation.MonitoredCounterGroup: Component type: SOURCE, name: r1 started
  396. 14/08/10 14:32:27 INFO source.AvroSource: Avro source r1 started.
  397. 14/08/10 14:32:36 INFO ipc.NettyServer: [id: 0x7c2f0aec, /192.168.1.50:38104 => /192.168.1.51:5555] OPEN
  398. 14/08/10 14:32:36 INFO ipc.NettyServer: [id: 0x7c2f0aec, /192.168.1.50:38104 => /192.168.1.51:5555] BOUND: /192.168.1.51:5555
  399. 14/08/10 14:32:36 INFO ipc.NettyServer: [id: 0x7c2f0aec, /192.168.1.50:38104 => /192.168.1.51:5555] CONNECTED: /192.168.1.50:38104
  400. 14/08/10 14:32:44 INFO ipc.NettyServer: [id: 0x3d36f553, /192.168.1.51:48599 => /192.168.1.51:5555] OPEN
  401. 14/08/10 14:32:44 INFO ipc.NettyServer: [id: 0x3d36f553, /192.168.1.51:48599 => /192.168.1.51:5555] BOUND: /192.168.1.51:5555
  402. 14/08/10 14:32:44 INFO ipc.NettyServer: [id: 0x3d36f553, /192.168.1.51:48599 => /192.168.1.51:5555] CONNECTED: /192.168.1.51:48599
  403. 14/08/10 14:34:33 INFO sink.LoggerSink: Event: { headers:{type=ali} body: 69 64 6F 61 6C 6C 5F 54 45 53 54 32 idoall_TEST2 }

  404. 复制代码

    　　　　可以看到，根据header中不同的条件分布到不同的channel上

    　　　　10)案例10：Flume Sink Processors
    　　　　failover的机器是一直发送给其中一个sink，当这个sink不可用的时候，自动发送到下一个sink。

    　　　　　　a)在m1创建Flume_Sink_Processors配置文件

  405. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/Flume_Sink_Processors.conf


    1. a1.sources = r1
  407. a1.sinks = k1 k2
  408. a1.channels = c1 c2

    1. #这个是配置failover的关键，需要有一个sink group
  410. a1.sinkgroups = g1
  411. a1.sinkgroups.g1.sinks = k1 k2
  412. #处理的类型是failover
  413. a1.sinkgroups.g1.processor.type = failover
  414. #优先级，数字越大优先级越高，每个sink的优先级必须不相同
  415. a1.sinkgroups.g1.processor.priority.k1 = 5
  416. a1.sinkgroups.g1.processor.priority.k2 = 10
  417. #设置为10秒，当然可以根据你的实际状况更改成更快或者很慢
  418. a1.sinkgroups.g1.processor.maxpenalty = 10000

    1. # Describe/configure the source
  420. a1.sources.r1.type = syslogtcp
  421. a1.sources.r1.port = 5140
  422. a1.sources.r1.channels = c1 c2
  423. a1.sources.r1.selector.type = replicating

    424. 24.
  425. # Describe the sink
  426. a1.sinks.k1.type = avro
  427. a1.sinks.k1.channel = c1
  428. a1.sinks.k1.hostname = m1
  429. a1.sinks.k1.port = 5555

    1. a1.sinks.k2.type = avro
  431. a1.sinks.k2.channel = c2
  432. a1.sinks.k2.hostname = m2
  433. a1.sinks.k2.port = 5555

    1. # Use a channel which buffers events in memory
  435. a1.channels.c1.type = memory
  436. a1.channels.c1.capacity = 1000
  437. a1.channels.c1.transactionCapacity = 100

    1. a1.channels.c2.type = memory
  439. a1.channels.c2.capacity = 1000

  440. a1.channels.c2.transactionCapacity = 100
    复制代码

    　　　　　　b)在m1创建Flume_Sink_Processors_avro配置文件

  441. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/Flume_Sink_Processors_avro.conf

    1. a1.sources = r1
  443. a1.sinks = k1
  444. a1.channels = c1

    1. # Describe/configure the source
  446. a1.sources.r1.type = avro
  447. a1.sources.r1.channels = c1
  448. a1.sources.r1.bind = 0.0.0.0
  449. a1.sources.r1.port = 5555

    1. # Describe the sink
  451. a1.sinks.k1.type = logger

    1. # Use a channel which buffers events in memory
  453. a1.channels.c1.type = memory
  454. a1.channels.c1.capacity = 1000
  455. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  457. a1.sources.r1.channels = c1

  458. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　c)将2个配置文件复制到m2上一份

  459. root@m1:/home/hadoop/flume-1.5.0-bin# scp -r /home/hadoop/flume-1.5.0-bin/conf/Flume_Sink_Processors.conf root@m2:/home/hadoop/flume-1.5.0-bin/conf/Flume_Sink_Processors.conf
  460. root@m1:/home/hadoop/flume-1.5.0-bin# scp -r /home/hadoop/flume-1.5.0-bin/conf/Flume_Sink_Processors_avro.conf root@m2:/home/hadoop/flume-1.5.0-bin/conf/Flume_Sink_Processors_avro.conf

  461. 复制代码

    　　　　　　d)打开4个窗口，在m1和m2上同时启动两个flume agent

  462. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/Flume_Sink_Processors_avro.conf -n a1 -Dflume.root.logger=INFO,console
  463. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/Flume_Sink_Processors.conf -n a1 -Dflume.root.logger=INFO,console

  464. 复制代码

    　　　　　　e)然后在m1或m2的任意一台机器上，测试产生log

  465. root@m1:/home/hadoop# echo “idoall.org test1 failover” | nc localhost 5140
    复制代码

    　　　　　　f)因为m2的优先级高，所以在m2的sink窗口，可以看到以下信息，而m1没有：

  466. 14/08/10 15:02:46 INFO ipc.NettyServer: Connection to /192.168.1.51:48692 disconnected.
  467. 14/08/10 15:03:12 INFO ipc.NettyServer: [id: 0x09a14036, /192.168.1.51:48704 => /192.168.1.51:5555] OPEN
  468. 14/08/10 15:03:12 INFO ipc.NettyServer: [id: 0x09a14036, /192.168.1.51:48704 => /192.168.1.51:5555] BOUND: /192.168.1.51:5555
  469. 14/08/10 15:03:12 INFO ipc.NettyServer: [id: 0x09a14036, /192.168.1.51:48704 => /192.168.1.51:5555] CONNECTED: /192.168.1.51:48704
  470. 14/08/10 15:03:26 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 69 64 6F 61 6C 6C 2E 6F 72 67 20 74 65 73 74 31 idoall.org test1 }

  471. 复制代码

    　　　　　　g)这时我们停止掉m2机器上的sink(ctrl+c)，再次输出测试数据：

  472. root@m1:/home/hadoop# echo “idoall.org test2 failover” | nc localhost 5140
    复制代码

    　　　　　　h)可以在m1的sink窗口，看到读取到了刚才发送的两条测试数据：

  473. 14/08/10 15:02:46 INFO ipc.NettyServer: Connection to /192.168.1.51:47036 disconnected.
  474. 14/08/10 15:03:12 INFO ipc.NettyServer: [id: 0xbcf79851, /192.168.1.51:47048 => /192.168.1.50:5555] OPEN
  475. 14/08/10 15:03:12 INFO ipc.NettyServer: [id: 0xbcf79851, /192.168.1.51:47048 => /192.168.1.50:5555] BOUND: /192.168.1.50:5555
  476. 14/08/10 15:03:12 INFO ipc.NettyServer: [id: 0xbcf79851, /192.168.1.51:47048 => /192.168.1.50:5555] CONNECTED: /192.168.1.51:47048
  477. 14/08/10 15:07:56 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 69 64 6F 61 6C 6C 2E 6F 72 67 20 74 65 73 74 31 idoall.org test1 }
  478. 14/08/10 15:07:56 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 69 64 6F 61 6C 6C 2E 6F 72 67 20 74 65 73 74 32 idoall.org test2 }

  479. 复制代码

    　　　　　　i)我们再在m2的sink窗口中，启动sink：

  480. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/Flume_Sink_Processors_avro.conf -n a1 -Dflume.root.logger=INFO,console

  481. 复制代码

    　　　　　　j)输入两批测试数据：

  482. root@m1:/home/hadoop# echo “idoall.org test3 failover” | nc localhost 5140 && echo “idoall.org test4 failover” | nc localhost 5140

  483. 复制代码

    　　　　　k)在m2的sink窗口，我们可以看到以下信息，因为优先级的关系，log消息会再次落到m2上：

  484. 14/08/10 15:09:47 INFO node.Application: Starting Sink k1
  485. 14/08/10 15:09:47 INFO node.Application: Starting Source r1
  486. 14/08/10 15:09:47 INFO source.AvroSource: Starting Avro source r1: { bindAddress: 0.0.0.0, port: 5555 }…
  487. 14/08/10 15:09:47 INFO instrumentation.MonitoredCounterGroup: Monitored counter group for type: SOURCE, name: r1: Successfully registered new MBean.
  488. 14/08/10 15:09:47 INFO instrumentation.MonitoredCounterGroup: Component type: SOURCE, name: r1 started
  489. 14/08/10 15:09:47 INFO source.AvroSource: Avro source r1 started.
  490. 14/08/10 15:09:54 INFO ipc.NettyServer: [id: 0x96615732, /192.168.1.51:48741 => /192.168.1.51:5555] OPEN
  491. 14/08/10 15:09:54 INFO ipc.NettyServer: [id: 0x96615732, /192.168.1.51:48741 => /192.168.1.51:5555] BOUND: /192.168.1.51:5555
  492. 14/08/10 15:09:54 INFO ipc.NettyServer: [id: 0x96615732, /192.168.1.51:48741 => /192.168.1.51:5555] CONNECTED: /192.168.1.51:48741
  493. 14/08/10 15:09:57 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 69 64 6F 61 6C 6C 2E 6F 72 67 20 74 65 73 74 32 idoall.org test2 }
  494. 14/08/10 15:10:43 INFO ipc.NettyServer: [id: 0x12621f9a, /192.168.1.50:38166 => /192.168.1.51:5555] OPEN
  495. 14/08/10 15:10:43 INFO ipc.NettyServer: [id: 0x12621f9a, /192.168.1.50:38166 => /192.168.1.51:5555] BOUND: /192.168.1.51:5555
  496. 14/08/10 15:10:43 INFO ipc.NettyServer: [id: 0x12621f9a, /192.168.1.50:38166 => /192.168.1.51:5555] CONNECTED: /192.168.1.50:38166
  497. 14/08/10 15:10:43 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 69 64 6F 61 6C 6C 2E 6F 72 67 20 74 65 73 74 33 idoall.org test3 }
  498. 14/08/10 15:10:43 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 69 64 6F 61 6C 6C 2E 6F 72 67 20 74 65 73 74 34 idoall.org test4 }

  499. 复制代码

    　　　　11)案例11：Load balancing Sink Processor
    　　　　load balance type和failover不同的地方是，load balance有两个配置，一个是轮询，一个是随机。两种情况下如果被选择的sink不可用，就会自动尝试发送到下一个可用的sink上面。

    　　　　　　a)在m1创建Load_balancing_Sink_Processors配置文件

  500. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/Load_balancing_Sink_Processors.conf


    1. a1.sources = r1
  502. a1.sinks = k1 k2
  503. a1.channels = c1

    1. #这个是配置Load balancing的关键，需要有一个sink group
  505. a1.sinkgroups = g1
  506. a1.sinkgroups.g1.sinks = k1 k2
  507. a1.sinkgroups.g1.processor.type = load_balance
  508. a1.sinkgroups.g1.processor.backoff = true
  509. a1.sinkgroups.g1.processor.selector = round_robin

    1. # Describe/configure the source
  511. a1.sources.r1.type = syslogtcp
  512. a1.sources.r1.port = 5140
  513. a1.sources.r1.channels = c1

    514. 19.
  515. # Describe the sink
  516. a1.sinks.k1.type = avro
  517. a1.sinks.k1.channel = c1
  518. a1.sinks.k1.hostname = m1
  519. a1.sinks.k1.port = 5555

    1. a1.sinks.k2.type = avro
  521. a1.sinks.k2.channel = c1
  522. a1.sinks.k2.hostname = m2
  523. a1.sinks.k2.port = 5555

    1. # Use a channel which buffers events in memory
  525. a1.channels.c1.type = memory
  526. a1.channels.c1.capacity = 1000

  527. a1.channels.c1.transactionCapacity = 100
    复制代码

    　　　　　　b)在m1创建Load_balancing_Sink_Processors_avro配置文件

  528. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/Load_balancing_Sink_Processors_avro.conf

    1. a1.sources = r1
  530. a1.sinks = k1
  531. a1.channels = c1

    1. # Describe/configure the source
  533. a1.sources.r1.type = avro
  534. a1.sources.r1.channels = c1
  535. a1.sources.r1.bind = 0.0.0.0
  536. a1.sources.r1.port = 5555

    1. # Describe the sink
  538. a1.sinks.k1.type = logger

    1. # Use a channel which buffers events in memory
  540. a1.channels.c1.type = memory
  541. a1.channels.c1.capacity = 1000
  542. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  544. a1.sources.r1.channels = c1

  545. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　c)将2个配置文件复制到m2上一份

  546. root@m1:/home/hadoop/flume-1.5.0-bin# scp -r /home/hadoop/flume-1.5.0-bin/conf/Load_balancing_Sink_Processors.conf root@m2:/home/hadoop/flume-1.5.0-bin/conf/Load_balancing_Sink_Processors.conf
  547. root@m1:/home/hadoop/flume-1.5.0-bin# scp -r /home/hadoop/flume-1.5.0-bin/conf/Load_balancing_Sink_Processors_avro.conf root@m2:/home/hadoop/flume-1.5.0-bin/conf/Load_balancing_Sink_Processors_avro.conf

  548. 复制代码

    　　　　　　d)打开4个窗口，在m1和m2上同时启动两个flume agent

  549. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/Load_balancing_Sink_Processors_avro.conf -n a1 -Dflume.root.logger=INFO,console

  550. root@m1:/home/hadoop# /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/Load_balancing_Sink_Processors.conf -n a1 -Dflume.root.logger=INFO,console

  551. 复制代码

    　　　　　　e)然后在m1或m2的任意一台机器上，测试产生log，一行一行输入，输入太快，容易落到一台机器上

  552. root@m1:/home/hadoop# echo “idoall.org test1” | nc localhost 5140
  553. root@m1:/home/hadoop# echo “idoall.org test2” | nc localhost 5140
  554. root@m1:/home/hadoop# echo “idoall.org test3” | nc localhost 5140

  555. root@m1:/home/hadoop# echo “idoall.org test4” | nc localhost 5140
    复制代码

    　　　　　　f)在m1的sink窗口，可以看到以下信息：

  556. 14/08/10 15:35:29 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 69 64 6F 61 6C 6C 2E 6F 72 67 20 74 65 73 74 32 idoall.org test2 }
  557. 14/08/10 15:35:33 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 69 64 6F 61 6C 6C 2E 6F 72 67 20 74 65 73 74 34 idoall.org test4 }

  558. 复制代码

    　　　　　　g)在m2的sink窗口，可以看到以下信息：

  559. 14/08/10 15:35:27 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 69 64 6F 61 6C 6C 2E 6F 72 67 20 74 65 73 74 31 idoall.org test1 }
  560. 14/08/10 15:35:29 INFO sink.LoggerSink: Event: { headers:{Severity=0, flume.syslog.status=Invalid, Facility=0} body: 69 64 6F 61 6C 6C 2E 6F 72 67 20 74 65 73 74 33 idoall.org test3 }

  561. 复制代码

    　　　　说明轮询模式起到了作用。

    　　　　12)案例12：Hbase sink

    　　　　　　a)在测试之前，请先参考《ubuntu12.04+hadoop2.2.0+zookeeper3.4.5+hbase0.96.2+hive0.13.1分布式环境部署》将hbase启动

    　　　　　　b)然后将以下文件复制到flume中：

  562. cp /home/hadoop/hbase-0.96.2-hadoop2/lib/protobuf-java-2.5.0.jar /home/hadoop/flume-1.5.0-bin/lib

  563. cp /home/hadoop/hbase-0.96.2-hadoop2/lib/hbase-client-0.96.2-hadoop2.jar /home/hadoop/flume-1.5.0-bin/lib
  564. cp /home/hadoop/hbase-0.96.2-hadoop2/lib/hbase-common-0.96.2-hadoop2.jar /home/hadoop/flume-1.5.0-bin/lib
  565. cp /home/hadoop/hbase-0.96.2-hadoop2/lib/hbase-protocol-0.96.2-hadoop2.jar /home/hadoop/flume-1.5.0-bin/lib
  566. cp /home/hadoop/hbase-0.96.2-hadoop2/lib/hbase-server-0.96.2-hadoop2.jar /home/hadoop/flume-1.5.0-bin/lib
  567. cp /home/hadoop/hbase-0.96.2-hadoop2/lib/hbase-hadoop2-compat-0.96.2-hadoop2.jar /home/hadoop/flume-1.5.0-bin/lib
  568. cp /home/hadoop/hbase-0.96.2-hadoop2/lib/hbase-hadoop-compat-0.96.2-hadoop2.jar /home/hadoop/flume-1.5.0-bin/lib@@@
  569. cp /home/hadoop/hbase-0.96.2-hadoop2/lib/htrace-core-2.04.jar /home/hadoop/flume-1.5.0-bin/lib

  570. 复制代码

    　　　　　　c)确保test_idoall_org表在hbase中已经存在

    　　　　　　d)在m1创建hbase_simple配置文件

  571. root@m1:/home/hadoop# vi /home/hadoop/flume-1.5.0-bin/conf/hbase_simple.conf

    1. a1.sources = r1
  573. a1.sinks = k1
  574. a1.channels = c1

    1. # Describe/configure the source
  576. a1.sources.r1.type = syslogtcp
  577. a1.sources.r1.port = 5140
  578. a1.sources.r1.host = localhost
  579. a1.sources.r1.channels = c1

    1. # Describe the sink
  581. a1.sinks.k1.type = logger
  582. a1.sinks.k1.type = hbase
  583. a1.sinks.k1.table = test_idoall_org
  584. a1.sinks.k1.columnFamily = name
  585. a1.sinks.k1.column = idoall
  586. a1.sinks.k1.serializer = org.apache.flume.sink.hbase.RegexHbaseEventSerializer
  587. a1.sinks.k1.channel = memoryChannel

    1. # Use a channel which buffers events in memory
  589. a1.channels.c1.type = memory
  590. a1.channels.c1.capacity = 1000
  591. a1.channels.c1.transactionCapacity = 100

    1. # Bind the source and sink to the channel
  593. a1.sources.r1.channels = c1

  594. a1.sinks.k1.channel = c1
    复制代码

    　　　　　　e)启动flume agent

  595. /home/hadoop/flume-1.5.0-bin/bin/flume-ng agent -c . -f /home/hadoop/flume-1.5.0-bin/conf/hbase_simple.conf -n a1 -Dflume.root.logger=INFO,console
    复制代码

    　　　　　　f)测试产生syslog

  596. root@m1:/home/hadoop# echo “hello idoall.org from flume” | nc localhost 5140
    复制代码

    　　　　　　g)这时登录到hbase中，可以发现新数据已经插入

  597. root@m1:/home/hadoop# /home/hadoop/hbase-0.96.2-hadoop2/bin/hbase shell
  598. 2014-08-10 16:09:48,984 INFO [main] Configuration.deprecation: hadoop.native.lib is deprecated. Instead, use io.native.lib.available
  599. HBase Shell; enter ‘help’ for list of supported commands.
  600. Type “exit” to leave the HBase Shell
  601. Version 0.96.2-hadoop2, r1581096, Mon Mar 24 16:03:18 PDT 2014

    1. hbase(main):001:0> list
  603. TABLE
  604. SLF4J: Class path contains multiple SLF4J bindings.
  605. SLF4J: Found binding in [jar:file:/home/hadoop/hbase-0.96.2-hadoop2/lib/slf4j-log4j12-1.6.4.jar!/org/slf4j/impl/StaticLoggerBinder.class]
  606. SLF4J: Found binding in [jar:file:/home/hadoop/hadoop-2.2.0/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
  607. SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
  608. hbase2hive_idoall
  609. hive2hbase_idoall
  610. test_idoall_org
  611. 3 row(s) in 2.6880 seconds

    1. => [“hbase2hive_idoall”, “hive2hbase_idoall”, “test_idoall_org”]
  613. hbase(main):002:0> scan “test_idoall_org”
  614. ROW COLUMN+CELL
  615. 10086 column=name:idoall, timestamp=1406424831473, value=idoallvalue
  616. 1 row(s) in 0.0550 seconds

    1. hbase(main):003:0> scan “test_idoall_org”
  618. ROW COLUMN+CELL
  619. 10086 column=name:idoall, timestamp=1406424831473, value=idoallvalue
  620. 1407658495588-XbQCOZrKK8-0 column=name:payload, timestamp=1407658498203, value=hello idoall.org from flume
  621. 2 row(s) in 0.0200 seconds

    1. hbase(main):004:0> quit
      复制代码

经过这么多flume的例子测试，如果你全部做完后，会发现flume的功能真的很强大，可以进行各种搭配来完成你想要的工作，俗话说师傅领进门，修行在个人，如何能够结合你的产品业务，将flume更好的应用起来，快去动手实践吧。

阅读全文
0 0
折八百淘宝商城马甲 折八百淘宝商城马甲
原创粉丝点击
热门IT博客
免费做广告的软件免费做广告的软件
apache和nginx配合apache和nginx配合
xmind mac子主题快捷键
ubuntu无法复制文件
c 调用js函数
公司电脑监控软件
sap sql
4g有信号无法连接网络
水印素材软件
淘宝外卖怎么入驻
淘宝人员考核标准
软件开发怎么学
免费房屋设计软件下载
融资融券系统源码
淘宝钻石等级买家
数据库存储过程是什么
淘宝网销售排行
天猫化妆品销售数据
淘宝网支持微信支付吗
连接linux的终端
热门问题 老师的惩罚 人脸识别 我在镇武司摸鱼那些年 重生之率土为王 我在大康的咸鱼生活 盘龙之生命进化 天生仙种 凡人之先天五行 春回大明朝 姑娘不必设防，我是瞎子 汇悦天地三期 汛组词 汛 桃花汛 汛怎么读 潮汛 汛拼音 汛的拼音 汛组词语 汛的组词 汛字组词 桃花汛旅游 汛怎么组词 潮汛的拼音 潮汛拼音 汛可以组什么词 汛的形近字 南宁恒汛网络科技有限公司 三江 三江人才网 三江锅 三江人才 三江学院 智慧三江 爆笑三江锅 三江平原 三江并流 建三江 三江口 三江购物 爆笑三江 三江侗族 三江风雨桥 三江县 宜宾三江 柳州三江 三江鼓楼 三江源头 三江指 三江指哪三江 广西三江

程序博客网，程序员的互联网技术博客家园。csdn论坛精品 msdn技术资料都在这里