flume收集日志到mysql

flume基本应用请参考:http://blog.csdn.net/liangrui1988/article/details/71774663

下面这个是将收集到的日志写到mysql中

github例子:https://github.com/liangrui1988/flume_mysql

pom:

<?xml version="1.0" encoding="UTF-8"?>  <project xmlns="http://maven.apache.org/POM/4.0.0"         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">  <modelVersion>4.0.0</modelVersion>  <groupId>com.flume.dome</groupId>  <artifactId>flume_mysql</artifactId>  <version>0.0.1</version><!--   <packaging>jar</packaging> -->  <name>flume_mysql</name>  <url>http://maven.apache.org</url>  <properties>    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>              <version.flume>1.7.0</version.flume>  </properties>      <dependencies>        <dependency>            <groupId>org.apache.flume</groupId>            <artifactId>flume-ng-core</artifactId>            <version>${version.flume}</version>        </dependency>        <dependency>            <groupId>org.apache.flume</groupId>            <artifactId>flume-ng-configuration</artifactId>            <version>${version.flume}</version>        </dependency>        <!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->        <dependency>            <groupId>mysql</groupId>            <artifactId>mysql-connector-java</artifactId>            <version>5.1.42</version>        </dependency>        <dependency><groupId>junit</groupId><artifactId>junit</artifactId><version>4.8.2</version><scope>test</scope></dependency>    </dependencies></project>

自定义一个sink写入到mysql: java代码

package com.flume.dome.mysink;import java.sql.Connection;import java.sql.DriverManager;import java.sql.PreparedStatement;import java.sql.SQLException;import java.util.List;import org.apache.flume.Channel;import org.apache.flume.Context;import org.apache.flume.Event;import org.apache.flume.EventDeliveryException;import org.apache.flume.Transaction;import org.apache.flume.conf.Configurable;import org.apache.flume.sink.AbstractSink;import org.mortbay.log.Log;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import com.google.common.base.Preconditions;import com.google.common.base.Throwables;import com.google.common.collect.Lists;public class MysqlSink extends AbstractSink implements Configurable {private Logger LOG = LoggerFactory.getLogger(MysqlSink.class);private String hostname;private String port;private String databaseName;private String tableName;private String user;private String password;private PreparedStatement preparedStatement;private Connection conn;private int batchSize;// 批处理数量public MysqlSink() {LOG.info("MysqlSink start...");}public Status process() throws EventDeliveryException {Status result = Status.READY;Channel channel = getChannel();Transaction transaction = channel.getTransaction();Event event;String content;// 数据集合List<String> actions = Lists.newArrayList();transaction.begin();try {for (int i = 0; i < batchSize; i++) {event = channel.take();// 从通道中获取数据if (event != null) {content = new String(event.getBody());actions.add(content);} else {result = Status.BACKOFF;break;}}if (actions.size() > 0) {preparedStatement.clearBatch();for (String temp : actions) {Log.info("actions temp:{}", temp);// 对占位符设置值，占位符顺序从1开始，第一个参数是占位符的位置，第二个参数是占位符的值。preparedStatement.setString(1, temp);preparedStatement.setLong(2, System.currentTimeMillis());preparedStatement.addBatch();}preparedStatement.executeBatch();conn.commit();}transaction.commit();} catch (Throwable e) {try {transaction.rollback();} catch (Exception e2) {LOG.error("Exception in rollback. Rollback might not have been"+ "successful.", e2);}LOG.error("Failed to commit transaction."+ "Transaction rolled back.", e);Throwables.propagate(e);} finally {transaction.close();}return result;}public void configure(Context context) {hostname = context.getString("hostname");Preconditions.checkNotNull(hostname, "hostname must be set!!");port = context.getString("port");Preconditions.checkNotNull(port, "port must be set!!");databaseName = context.getString("databaseName");Preconditions.checkNotNull(databaseName, "databaseName must be set!!");tableName = context.getString("tableName");Preconditions.checkNotNull(tableName, "tableName must be set!!");user = context.getString("user");Preconditions.checkNotNull(user, "user must be set!!");password = context.getString("password");Preconditions.checkNotNull(password, "password must be set!!");batchSize = context.getInteger("batchSize", 100);Preconditions.checkNotNull(batchSize > 0,"batchSize must be a positive number!!");}@Overridepublic synchronized void start() {super.start();try {// 调用Class.forName()方法加载驱动程序Class.forName("com.mysql.jdbc.Driver");} catch (ClassNotFoundException e) {e.printStackTrace();}String url = "jdbc:mysql://" + hostname + ":" + port + "/"+ databaseName;Log.info("mysql start url:{}", url);// 调用DriverManager对象的getConnection()方法，获得一个Connection对象try {conn = DriverManager.getConnection(url, user, password);conn.setAutoCommit(false);// 创建一个Statement对象preparedStatement = conn.prepareStatement("insert into "+ tableName + " (context,time) values (?,?)");} catch (SQLException e) {e.printStackTrace();System.exit(1);}}@Overridepublic synchronized void stop() {super.stop();if (preparedStatement != null) {try {preparedStatement.close();} catch (SQLException e) {e.printStackTrace();}}if (conn != null) {try {conn.close();} catch (SQLException e) {e.printStackTrace();}}}}

mvn clean 
mvn install
将代码打成jar包后，上传到flume安装目录下的lib文件夹中，同时需要上传MySQL的驱动jar包

-------------测试 Spooling Directory Source--------------------------
conf：mysql_sink.conf

# Name the components on this agenta1.sources = r1a1.sinks = k1a1.channels = c1# Describe/configure the sourcea1.sources.r1.type = spooldira1.sources.r1.spoolDir = /home/rui/log/flumespoola1.sources.r1.fileHeader = truea1.sources.r1.channels = c1# Describe the sinka1.sinks.k1.type = com.flume.dome.mysink.MysqlSinka1.sinks.k1.hostname = 192.168.254.1a1.sinks.k1.port = 3306a1.sinks.k1.databaseName = flume_dba1.sinks.k1.tableName = log_dataa1.sinks.k1.user = lianga1.sinks.k1.password = 123456a1.sinks.k1.channel = c1# Use a channel which buffers events in memorya1.channels.c1.type = memorya1.channels.c1.capacity = 1000a1.channels.c1.transactionCapacity = 100

启动flume agent al
bin/flume-ng agent --conf conf --conf-file conf/mysql_sink.conf --name a1 -Dflume.root.logger=INFO,console

创间表：

CREATE TABLE `log_data` (`id`  int(11) NOT NULL AUTO_INCREMENT ,`context`  varchar(255) CHARACTER SET latin1 COLLATE latin1_swedish_ci NULL DEFAULT NULL ,`time`  bigint(20) NULL DEFAULT NULL ,PRIMARY KEY (`id`))ENGINE=InnoDBDEFAULT CHARACTER SET=latin1 COLLATE=latin1_swedish_ciAUTO_INCREMENT=1ROW_FORMAT=COMPACT;

测试:
cd /home/rui/log/flumespool
echo "hello world" > test.log

数据写入成功!