flume自定义sink

来源:互联网 发布:js new york 编辑:程序博客网 时间:2024/06/06 07:37

java文件:

package com.dle;import java.io.File;import java.io.FileNotFoundException;import java.io.FileOutputStream;import java.io.IOException;import org.apache.flume.Channel;import org.apache.flume.Context;import org.apache.flume.Event;import org.apache.flume.EventDeliveryException;import org.apache.flume.Transaction;import org.apache.flume.conf.Configurable;import org.apache.flume.sink.AbstractSink;import org.slf4j.Logger;import org.slf4j.LoggerFactory;public class MySink extends AbstractSink implements Configurable {    private static final Logger logger = LoggerFactory.getLogger(MySink.class);    private static final String PROP_KEY_ROOTPATH = "fileName";    private String fileName;    @Override    public void configure(Context context) {        // TODO Auto-generated method stub        fileName = context.getString(PROP_KEY_ROOTPATH);    }    @Override    public Status process() throws EventDeliveryException {        // TODO Auto-generated method stub        Channel ch = getChannel();        Transaction txn = ch.getTransaction();        Event event =null;        txn.begin();        while(true){            event = ch.take();            if (event!=null) {                break;            }        }        try {            logger.debug("Get event.");            String body = new String(event.getBody());            String res = body + ":" + System.currentTimeMillis() + "\r\n";            File file = new File(fileName);            FileOutputStream fos = null;            try {                fos = new FileOutputStream(file, true);            } catch (FileNotFoundException e) {                // TODO Auto-generated catch block                e.printStackTrace();            }            try {                fos.write(res.getBytes());            } catch (IOException e) {                // TODO Auto-generated catch block                e.printStackTrace();            }            try {                fos.close();            } catch (IOException e) {                // TODO Auto-generated catch block                e.printStackTrace();            }            txn.commit();            return Status.READY;        } catch (Throwable th) {            txn.rollback();            if (th instanceof Error) {                throw (Error) th;            } else {                throw new EventDeliveryException(th);            }        } finally {            txn.close();        }    }}

配置信息:

a1.sources = r1a1.sinks = k1a1.channels = c1# Describe/configure the sourcea1.sources.r1.type = execa1.sources.r1.channels = c1a1.sources.r1.command = tail -F /tmp/test/a.txt# Describe the sinka1.sinks.k1.type = com.dle.MySinka1.sinks.k1.fileName=/tmp/test/target.txt# Use a channel which buffers events in memorya1.channels.c1.type = memorya1.channels.c1.capacity = 1000a1.channels.c1.transactionCapacity = 100# Bind the source and sink to the channela1.sources.r1.channels = c1a1.sinks.k1.channel = c1
0 0