Flume NG与MySQL整合开发

来源:互联网 发布:人工智能百度云资源 编辑:程序博客网 时间:2024/06/10 16:12

1 pom文件配置

<dependencies>    <dependency>        <groupId>org.apache.flume</groupId>        <artifactId>flume-ng-configuration</artifactId>        <version>1.5.2</version>    </dependency>    <dependency>        <groupId>org.apache.flume</groupId>        <artifactId>flume-ng-core</artifactId>        <version>1.5.2</version>    </dependency>    <dependency>        <groupId>mysql</groupId>        <artifactId>mysql-connector-java</artifactId>        <version>5.1.26</version>    </dependency>    <dependency>        <groupId>com.alibaba</groupId>        <artifactId>druid</artifactId>        <version>1.0.27</version>    </dependency></dependencies>

2 java代码实现

package cn.gcks;import java.sql.Connection;import java.sql.DriverManager;import java.sql.PreparedStatement;import java.sql.SQLException;import java.util.List;import java.util.regex.Pattern;import org.apache.flume.Channel;import org.apache.flume.Context;import org.apache.flume.Event;import org.apache.flume.EventDeliveryException;import org.apache.flume.Transaction;import org.apache.flume.conf.Configurable;import org.apache.flume.sink.AbstractSink;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import com.google.common.base.Preconditions;import com.google.common.base.Throwables;import com.google.common.collect.Lists;/** * Flume收集文件到MySQL *  * @author 林志明 * */public class MySQLSink extends AbstractSink implements Configurable {    private Logger logger = LoggerFactory.getLogger(MySQLSink.class);    private String hostname;    private String port;    private String databaseName;    private String tableName;    private String user;    private String password;    private PreparedStatement preparedStatement;    private Connection conn;    private int batchSize;    private String columnName;    private String fieldSeparator;    private int columnNumber;    public MySQLSink() {        logger.info("MysqlSink start...");    }    Pattern pattern = null;    @Override    public void configure(Context context) {        hostname = context.getString("hostname");        Preconditions.checkNotNull(hostname, "hostname must be set!!");        port = context.getString("port");        Preconditions.checkNotNull(port, "port must be set!!");        databaseName = context.getString("databaseName");        Preconditions.checkNotNull(databaseName, "databaseName must be set!!");        tableName = context.getString("tableName");        Preconditions.checkNotNull(tableName, "tableName must be set!!");        user = context.getString("user");        Preconditions.checkNotNull(user, "user must be set!!");        password = context.getString("password");        Preconditions.checkNotNull(password, "password must be set!!");        batchSize = context.getInteger("batchSize", 100);        Preconditions.checkNotNull(batchSize > 0, "batchSize must be a positive number!!");        columnName = context.getString("column_name");        Preconditions.checkNotNull(columnName, "column_name must be set!!");        fieldSeparator = context.getString("field_separator");        Preconditions.checkNotNull(fieldSeparator, "field_separator must be set!!");        pattern = Pattern.compile(fieldSeparator);    }    private String sql = null;    @Override    public void start() {        super.start();        String url = CommonUtils.append("jdbc:mysql://", hostname, ":", port, "/", databaseName,                "?Unicode=true&characterEncoding=UTF-8");        try {            conn = DataSourceFactory.getConnection(url, user, password);            conn.setAutoCommit(false);            // 创建一个Statement对象            String[] columns = columnName.split(",");            String values = null;            StringBuffer stringBuffer = new StringBuffer();            int i = 0;            columnNumber = columns.length;            while (i < columnNumber) {                stringBuffer.append("?").append(",");                i++;            }            values = stringBuffer.substring(0, stringBuffer.length() - 1).toString();            sql = "insert into " + tableName + " (" + columnName + ") values (" + values + ")";        } catch (SQLException e) {            e.printStackTrace();            System.exit(1);        }    }    @Override    public void stop() {        super.stop();        DataSourceFactory.closeCon(preparedStatement, conn);    }    @Override    public Status process() throws EventDeliveryException {        try {            preparedStatement = conn.prepareStatement(sql);            logger.info("===preparedStatement Alive===");        } catch (SQLException e1) {            e1.printStackTrace();        }        Status result = Status.READY;        Channel channel = getChannel();        Transaction transaction = channel.getTransaction();        Event event;        String content;        List<String> actions = Lists.newArrayList();        transaction.begin();        try {            for (int i = 0; i < batchSize; i++) {                event = channel.take();                if (event != null) {                    content = new String(event.getBody());                    actions.add(content);                } else {                    result = Status.BACKOFF;                    break;                }            }            if (actions.size() > 0) {                preparedStatement.clearBatch();                for (String temp : actions) {                    String[] datas = {};                    if (temp != null && temp.trim().length() != 0) {                        datas = pattern.split(temp);                    }                    for (int i = 0; i < columnNumber; i++) {                        if (i > datas.length - 1) {                            preparedStatement.setString(i + 1, null);                        } else {                            preparedStatement.setString(i + 1, datas[i]);                        }                    }                    preparedStatement.addBatch();                }                preparedStatement.executeBatch();                conn.commit();            }            transaction.commit();        } catch (Throwable e) {            try {                transaction.rollback();            } catch (Exception e2) {                logger.error("Exception in rollback. Rollback might not have been" + "successful.", e2);            }            logger.error("Failed to commit transaction." + "Transaction rolled back.", e);            Throwables.propagate(e);        } finally {            transaction.close();        }        return result;    }}
package cn.gcks;import java.sql.Connection;import java.sql.PreparedStatement;import java.sql.ResultSet;import org.apache.log4j.Logger;import com.alibaba.druid.pool.DruidDataSource;/** * 数据库连接 *  * @author linzm * */public class DataSourceFactory {    private static Logger log = Logger.getLogger(DataSourceFactory.class);    private static DruidDataSource dataSource = null;    /**     * 创建数据源     *      * @return     */    public static DruidDataSource getDataSource(String url, String userName, String password) throws Exception {        if (dataSource == null) {            log.info("数据库连接信息:[url:" + url + ",userName:" + userName + ",password:" + password + "]");            dataSource = new DruidDataSource();            dataSource.setUrl(url);            dataSource.setUsername(userName);            dataSource.setPassword(password);            dataSource.setMaxActive(15);// 设置最大并发数            dataSource.setInitialSize(2);// 数据库初始化时,创建的连接个数            dataSource.setMaxWait(60000);            dataSource.setMinIdle(1);// 最小空闲连接数            dataSource.setTimeBetweenEvictionRunsMillis(5 * 60 * 1000);// 5分钟检测一次是否有死掉的线程            dataSource.setMinEvictableIdleTimeMillis(300000);// 空闲连接60秒中后释放            dataSource.setTestWhileIdle(true);            // 检测连接有效性            dataSource.setTestOnBorrow(true);            dataSource.setValidationQuery("select 1");            dataSource.setPoolPreparedStatements(true);            dataSource.setMaxOpenPreparedStatements(15);        }        return dataSource;    }    /**     * 释放数据源     */    public static void shutDownDataSource() throws Exception {        if (dataSource != null) {            dataSource.close();        }    }    /**     * 获取数据库连接     *      * @return     */    public static Connection getConnection(String url, String userName, String password) {        Connection con = null;        try {            if (dataSource != null) {                con = dataSource.getConnection();            } else {                con = getDataSource(url, userName, password).getConnection();            }        } catch (Exception e) {            log.error(e.getMessage(), e);        }        return con;    }    /**     * 关闭连接     */    public static void closeCon(PreparedStatement ps, Connection con) {        if (ps != null) {            try {                ps.close();            } catch (Exception e) {                log.error("预编译SQL语句对象PreparedStatement关闭异常!" + e.getMessage(), e);            }        }        if (con != null) {            try {                con.close();            } catch (Exception e) {                log.error("关闭连接对象Connection异常!" + e.getMessage(), e);            }        }    }}
package cn.gcks;import java.text.ParseException;import java.text.SimpleDateFormat;import java.util.Date;import java.util.Locale;import java.util.regex.Matcher;import java.util.regex.Pattern;public class CommonUtils {    public static final String EMPTY = "";    /**     * Represents a failed index search.     *     * @since 2.1     */    public static final int INDEX_NOT_FOUND = -1;    /**     * <p>     * The maximum size to which the padding constant(s) can expand.     * </p>     */    // private static final int PAD_LIMIT = 8192;    /**     * <p>     * Checks if a String is whitespace, empty ("") or null.     * </p>     * <p>     * <pre>     * StringUtils.isBlank(null)      = true     * StringUtils.isBlank("")        = true     * StringUtils.isBlank(" ")       = true     * StringUtils.isBlank("bob")     = false     * StringUtils.isBlank("  bob  ") = false     * </pre>     *     * @param str the String to check, may be null     * @return <code>true</code> if the String is null, empty or whitespace     * @since 2.0     */    public static boolean isBlank(String str) {        int strLen;        if (str == null || (strLen = str.length()) == 0) {            return true;        }        for (int i = 0; i < strLen; i++) {            if ((Character.isWhitespace(str.charAt(i)) == false)) {                return false;            }        }        return true;    }    /**     * <p>     * Checks if a String is whitespace, empty ("") or null.     * </p>     * <p>     * <pre>     * StringUtils.isBlank(null)      = true     * StringUtils.isBlank("")        = true     * StringUtils.isBlank(" ")       = true     * StringUtils.isBlank("bob")     = false     * StringUtils.isBlank("  bob  ") = false     * </pre>     *     * @param str the String to check, may be null     * @return <code>true</code> if the String is null, empty or whitespace     * @since 2.0     */    public static boolean isNotBlank(String str) {        return !isBlank(str);    }    public static Long parseLong(String str) {        try {            return Long.parseLong(str);        } catch (Exception ex) {            return 0L;        }    }    public static String toCNDateTimeString(Date date) {        SimpleDateFormat sf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");        return sf.format(date);    }    public static int strLengthContainBlank(String str) {        return isBlank(str) ? 0 : str.length();    }    /**     * <p>     * Gets a substring from the specified String avoiding exceptions.     * </p>     * <p>     * <p>     * A negative start position can be used to start/end <code>n</code>     * characters from the end of the String.     * </p>     * <p>     * <p>     * The returned substring starts with the character in the     * <code>start</code> position and ends before the <code>end</code>     * position. All position counting is zero-based -- i.e., to start at the     * beginning of the string use <code>start = 0</code>. Negative start and     * end positions can be used to specify offsets relative to the end of the     * String.     * </p>     * <p>     * <p>     * If <code>start</code> is not strictly to the left of <code>end</code>, ""     * is returned.     * </p>     * <p>     * <pre>     * StringUtils.substring(null, *, *)    = null     * StringUtils.substring("", * ,  *)    = "";     * StringUtils.substring("abc", 0, 2)   = "ab"     * StringUtils.substring("abc", 2, 0)   = ""     * StringUtils.substring("abc", 2, 4)   = "c"     * StringUtils.substring("abc", 4, 6)   = ""     * StringUtils.substring("abc", 2, 2)   = ""     * StringUtils.substring("abc", -2, -1) = "b"     * StringUtils.substring("abc", -4, 2)  = "ab"     * </pre>     *     * @param str   the String to get the substring from, may be null     * @param start the position to start from, negative means count back from the     *              end of the String by this many characters     * @param end   the position to end at (exclusive), negative means count back     *              from the end of the String by this many characters     * @return substring from start position to end positon, <code>null</code>     * if null String input     */    public static String substring(String str, int start, int end) {        if (str == null) {            return null;        }        // handle negatives        if (end < 0) {            end = str.length() + end; // remember end is negative        }        if (start < 0) {            start = str.length() + start; // remember start is negative        }        // check length next        if (end > str.length()) {            end = str.length();        }        // if start is greater than end, return ""        if (start > end) {            return EMPTY;        }        if (start < 0) {            start = 0;        }        if (end < 0) {            end = 0;        }        return str.substring(start, end);    }    public static String append(String... strings) {        StringBuffer stringBuffer = new StringBuffer();        for (String string : strings) {            stringBuffer.append(string);        }        return stringBuffer.toString();    }    public static String appendWithOperator(String operator, Object... objs) {        StringBuffer stringBuffer = new StringBuffer();        for (Object obj : objs) {            if (obj instanceof Object[]) {                Object[] arr = (Object[]) obj;                for (Object obj2 : arr) {                    stringBuffer.append(obj2).append(operator);                }            } else {                stringBuffer.append(obj).append(operator);            }        }        return substring(stringBuffer.toString(), 0, -1);    }    public static String appendWithOperatorReplaceString(String operator, String nullReplaceString, Object... objs) {        StringBuffer stringBuffer = new StringBuffer();        for (Object obj : objs) {            if (obj != null && isBlank(obj.toString())) {                obj = nullReplaceString;            }            if (obj instanceof Object[]) {                Object[] arr = (Object[]) obj;                for (Object obj2 : arr) {                    if (obj2 == null) {                        obj2 = nullReplaceString;                    }                    stringBuffer.append(obj2).append(operator);                }            } else {                stringBuffer.append(obj).append(operator);            }        }        return substring(stringBuffer.toString(), 0, -1);    }    public static boolean equals(String str1, String str2) {        return str1 == null ? str2 == null : str1.equals(str2);    }    public static boolean notEquals(String str1, String str2) {        return !(str1 == null ? str2 == null : str1.equals(str2));    }    /**     * <p>     * Checks if String contains a search String, handling <code>null</code>.     * This method uses {@link String#indexOf(String)}.     * </p>     * <p>     * <p>     * A <code>null</code> String will return <code>false</code>.     * </p>     * <p>     * <pre>     * StringUtils.contains(null, *)     = false     * StringUtils.contains(*, null)     = false     * StringUtils.contains("", "")      = true     * StringUtils.contains("abc", "")   = true     * StringUtils.contains("abc", "a")  = true     * StringUtils.contains("abc", "z")  = false     * </pre>     *     * @param str       the String to check, may be null     * @param searchStr the String to find, may be null     * @return true if the String contains the search String, false if not or     * <code>null</code> string input     * @since 2.0     */    public static boolean contains(String str, String searchStr) {        if (str == null || searchStr == null) {            return false;        }        return str.indexOf(searchStr) >= 0;    }    public static String[] split(String str, String operator, Pattern pattern) {        if (str == null || operator == null) {            return null;        } else {            if (contains(str, operator)) {                return pattern.split(str);            } else {                return null;            }        }    }    // StripAll    // -----------------------------------------------------------------------    /**     * <p>     * Strips whitespace from the start and end of every String in an array.     * Whitespace is defined by {@link Character#isWhitespace(char)}.     * </p>     * <p>     * <p>     * A new array is returned each time, except for length zero. A     * <code>null</code> array will return <code>null</code>. An empty array     * will return itself. A <code>null</code> array entry will be ignored.     * </p>     * <p>     * <pre>     * StringUtils.stripAll(null)             = null     * StringUtils.stripAll([])               = []     * StringUtils.stripAll(["abc", "  abc"]) = ["abc", "abc"]     * StringUtils.stripAll(["abc  ", null])  = ["abc", null]     * </pre>     *     * @param strs the array to remove whitespace from, may be null     * @return the stripped Strings, <code>null</code> if null array input     */    public static String[] stripAll(String[] strs) {        return stripAll(strs, null);    }    /**     * <p>     * Strips any of a set of characters from the start and end of every String     * in an array.     * </p>     * Whitespace is defined by {@link Character#isWhitespace(char)}.     * </p>     * <p>     * <p>     * A new array is returned each time, except for length zero. A     * <code>null</code> array will return <code>null</code>. An empty array     * will return itself. A <code>null</code> array entry will be ignored. A     * <code>null</code> stripChars will strip whitespace as defined by     * {@link Character#isWhitespace(char)}.     * </p>     * <p>     * <pre>     * StringUtils.stripAll(null, *)                = null     * StringUtils.stripAll([], *)                  = []     * StringUtils.stripAll(["abc", "  abc"], null) = ["abc", "abc"]     * StringUtils.stripAll(["abc  ", null], null)  = ["abc", null]     * StringUtils.stripAll(["abc  ", null], "yz")  = ["abc  ", null]     * StringUtils.stripAll(["yabcz", null], "yz")  = ["abc", null]     * </pre>     *     * @param strs       the array to remove characters from, may be null     * @param stripChars the characters to remove, null treated as whitespace     * @return the stripped Strings, <code>null</code> if null array input     */    public static String[] stripAll(String[] strs, String stripChars) {        int strsLen;        if (strs == null || (strsLen = strs.length) == 0) {            return strs;        }        String[] newArr = new String[strsLen];        for (int i = 0; i < strsLen; i++) {            newArr[i] = strip(strs[i], stripChars);        }        return newArr;    }    /**     * <p>     * Strips any of a set of characters from the start and end of a String.     * This is similar to {@link String#trim()} but allows the characters to be     * stripped to be controlled.     * </p>     * <p>     * <p>     * A <code>null</code> input String returns <code>null</code>. An empty     * string ("") input returns the empty string.     * </p>     * <p>     * <p>     * If the stripChars String is <code>null</code>, whitespace is stripped as     * defined by {@link Character#isWhitespace(char)}. Alternatively use     * {@link #strip(String)}.     * </p>     * <p>     * <pre>     * StringUtils.strip(null, *)          = null     * StringUtils.strip("", *)            = ""     * StringUtils.strip("abc", null)      = "abc"     * StringUtils.strip("  abc", null)    = "abc"     * StringUtils.strip("abc  ", null)    = "abc"     * StringUtils.strip(" abc ", null)    = "abc"     * StringUtils.strip("  abcyx", "xyz") = "  abc"     * </pre>     *     * @param str        the String to remove characters from, may be null     * @param stripChars the characters to remove, null treated as whitespace     * @return the stripped String, <code>null</code> if null String input     */    public static String strip(String str, String stripChars) {        if (isEmpty(str)) {            return str;        }        str = stripStart(str, stripChars);        return stripEnd(str, stripChars);    }    /**     * <p>     * Strips any of a set of characters from the start of a String.     * </p>     * <p>     * <p>     * A <code>null</code> input String returns <code>null</code>. An empty     * string ("") input returns the empty string.     * </p>     * <p>     * <p>     * If the stripChars String is <code>null</code>, whitespace is stripped as     * defined by {@link Character#isWhitespace(char)}.     * </p>     * <p>     * <pre>     * StringUtils.stripStart(null, *)          = null     * StringUtils.stripStart("", *)            = ""     * StringUtils.stripStart("abc", "")        = "abc"     * StringUtils.stripStart("abc", null)      = "abc"     * StringUtils.stripStart("  abc", null)    = "abc"     * StringUtils.stripStart("abc  ", null)    = "abc  "     * StringUtils.stripStart(" abc ", null)    = "abc "     * StringUtils.stripStart("yxabc  ", "xyz") = "abc  "     * </pre>     *     * @param str        the String to remove characters from, may be null     * @param stripChars the characters to remove, null treated as whitespace     * @return the stripped String, <code>null</code> if null String input     */    public static String stripStart(String str, String stripChars) {        int strLen;        if (str == null || (strLen = str.length()) == 0) {            return str;        }        int start = 0;        if (stripChars == null) {            while ((start != strLen) && Character.isWhitespace(str.charAt(start))) {                start++;            }        } else if (stripChars.length() == 0) {            return str;        } else {            while ((start != strLen) && (stripChars.indexOf(str.charAt(start)) != INDEX_NOT_FOUND)) {                start++;            }        }        return str.substring(start);    }    /**     * <p>     * Strips any of a set of characters from the end of a String.     * </p>     * <p>     * <p>     * A <code>null</code> input String returns <code>null</code>. An empty     * string ("") input returns the empty string.     * </p>     * <p>     * <p>     * If the stripChars String is <code>null</code>, whitespace is stripped as     * defined by {@link Character#isWhitespace(char)}.     * </p>     * <p>     * <pre>     * StringUtils.stripEnd(null, *)          = null     * StringUtils.stripEnd("", *)            = ""     * StringUtils.stripEnd("abc", "")        = "abc"     * StringUtils.stripEnd("abc", null)      = "abc"     * StringUtils.stripEnd("  abc", null)    = "  abc"     * StringUtils.stripEnd("abc  ", null)    = "abc"     * StringUtils.stripEnd(" abc ", null)    = " abc"     * StringUtils.stripEnd("  abcyx", "xyz") = "  abc"     * StringUtils.stripEnd("120.00", ".0")   = "12"     * </pre>     *     * @param str        the String to remove characters from, may be null     * @param stripChars the set of characters to remove, null treated as whitespace     * @return the stripped String, <code>null</code> if null String input     */    public static String stripEnd(String str, String stripChars) {        int end;        if (str == null || (end = str.length()) == 0) {            return str;        }        if (stripChars == null) {            while ((end != 0) && Character.isWhitespace(str.charAt(end - 1))) {                end--;            }        } else if (stripChars.length() == 0) {            return str;        } else {            while ((end != 0) && (stripChars.indexOf(str.charAt(end - 1)) != INDEX_NOT_FOUND)) {                end--;            }        }        return str.substring(0, end);    }    // Empty checks    // -----------------------------------------------------------------------    /**     * <p>     * Checks if a String is empty ("") or null.     * </p>     * <p>     * <pre>     * StringUtils.isEmpty(null)      = true     * StringUtils.isEmpty("")        = true     * StringUtils.isEmpty(" ")       = false     * StringUtils.isEmpty("bob")     = false     * StringUtils.isEmpty("  bob  ") = false     * </pre>     * <p>     * <p>     * NOTE: This method changed in Lang version 2.0. It no longer trims the     * String. That functionality is available in isBlank().     * </p>     *     * @param str the String to check, may be null     * @return <code>true</code> if the String is empty or null     */    public static boolean isEmpty(String str) {        return str == null || str.length() == 0;    }    // Stripping    // -----------------------------------------------------------------------    /**     * <p>     * Strips whitespace from the start and end of a String.     * </p>     * <p>     * <p>     * This is similar to {@link (String)} but removes whitespace.     * Whitespace is defined by {@link Character#isWhitespace(char)}.     * </p>     * <p>     * <p>     * A <code>null</code> input String returns <code>null</code>.     * </p>     * <p>     * <pre>     * StringUtils.strip(null)     = null     * StringUtils.strip("")       = ""     * StringUtils.strip("   ")    = ""     * StringUtils.strip("abc")    = "abc"     * StringUtils.strip("  abc")  = "abc"     * StringUtils.strip("abc  ")  = "abc"     * StringUtils.strip(" abc ")  = "abc"     * StringUtils.strip(" ab c ") = "ab c"     * </pre>     *     * @param str the String to remove whitespace from, may be null     * @return the stripped String, <code>null</code> if null String input     */    public static String strip(String str) {        return strip(str, null);    }    /**     * <p>     * Converts a String to lower case as per {@link String#toLowerCase()}.     * </p>     * <p>     * <p>     * A <code>null</code> input String returns <code>null</code>.     * </p>     * <p>     * <pre>     * StringUtils.lowerCase(null)  = null     * StringUtils.lowerCase("")    = ""     * StringUtils.lowerCase("aBc") = "abc"     * </pre>     * <p>     * <p>     * <strong>Note:</strong> As described in the documentation for     * {@link String#toLowerCase()}, the result of this method is affected by     * the current locale. For platform-independent case transformations, the     * method {@link (String, Locale)} should be used with a specific     * locale (e.g. {@link Locale#ENGLISH}).     * </p>     *     * @param str the String to lower case, may be null     * @return the lower cased String, <code>null</code> if null String input     */    public static String toLowerCase(String str) {        if (isBlank(str)) {            return str;        }        return str.toLowerCase();    }    public static String normalDateToConnectorDate (String time){        SimpleDateFormat format =  new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");        SimpleDateFormat format2 =  new SimpleDateFormat("yyyyMMddHHmmss");        Date date = null;        try {            date = format.parse(time);        } catch (ParseException e) {            e.toString();            return null;        }        return format2.format(date);    }}

3 打包java代码,上传jar包和mysql驱动包到flume的lib目录
进入源码的根目录执行mvn install -DskipTests -Dtar
4 创建MySQL表

CREATE TABLE `test_flume_sink` (  `id` int(11) NOT NULL AUTO_INCREMENT,  `content` varchar(255) DEFAULT NULL,  `log2` bigint(25) DEFAULT NULL,  PRIMARY KEY (`id`)) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

5 修改flume配置文件

agent.sinks.mysqlSink.type = cn.bfire.MySQLSinkagent.sinks.mysqlSink.hostname=1.1.1.1agent.sinks.mysqlSink.port=3306agent.sinks.mysqlSink.databaseName=bigdataagent.sinks.mysqlSink.tableName=test_flume_sinkagent.sinks.mysqlSink.user=miagent.sinks.mysqlSink.password=11111111agent.sinks.mysqlSink.column_name=content,log2agent.sinks.mysqlSink.field_separator=\\|

6 flume收集文件内容

abc|dddeee|fffaaa,gggddd|fffsss|jjjfff,fffaaaaaa|gg|fffffaaaaa,bbbbbbbbbb,cccccdddddd|eeeeeee|aaxxxx|eeeezzz|aaa,bbb|xxzz

7 MySQL表收集结果
MySQL表收集结果
8 遇到的问题【如果使用flume1.6.0下面的内容属于无聊】

错误日志:

java.lang.IllegalStateException: File should not roll when commit is outstanding.    at org.apache.flume.client.avro.ReliableSpoolingFileEventReader.readEvents(ReliableSpoolingFileEventReader.java:221)    at org.apache.flume.source.SpoolDirectorySource$SpoolDirectoryRunnable.run(SpoolDirectorySource.java:227)    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)    at java.lang.Thread.run(Thread.java:745)

原因 : flume收集目录中有文件大小为0
这里写图片描述
不过,这是flume1.6.0之前版本的BUG
https://issues.apache.org/jira/browse/FLUME-1934
http://flume.apache.org/releases/1.6.0.html
这里写图片描述

0 0
原创粉丝点击