自定义flume-source

来源：互联网发布：linux server 版本编辑：程序博客网时间：2024/06/08 04:17

package com.bigdata.flume;import org.apache.flume.Context;import org.apache.flume.Event;import org.apache.flume.EventDeliveryException;import org.apache.flume.PollableSource;import org.apache.flume.conf.Configurable;import org.apache.flume.event.EventBuilder;import org.apache.flume.source.AbstractSource;import java.nio.charset.Charset;import java.util.HashMap;import java.util.Random;/** *  */public class MySource extends AbstractSource implements Configurable, PollableSource {    @Override    public void configure(Context context) {    }    @Override    public Status process() throws EventDeliveryException {        Status status = null;        try{            Random random = new Random();            int randomNum = random.nextInt(100);            String text = "Hello World :" + random.nextInt(100);            HashMap<String, String> header = new HashMap<String, String>();            header.put("id", random+"");            this.getChannelProcessor()                    .processEvent(EventBuilder.withBody(text, Charset.forName("UTF-8"), header));            status = Status.READY;        }catch (Throwable t) {            // Log exception, handle individual exceptions as needed            status = Status.BACKOFF;            // re-throw all Errors            if (t instanceof Error) {                throw (Error)t;            }        }        return status;    }}

2.flume-source.conf

#a1.sources=r1a1.sinks=k1a1.channels=c1  #a1.sources.r1.type=com.bigdata.flume.MySource  #a1.channels.c1.type=memorya1.channels.c1.capacity=1000a1.channels.c1.transactionCapacity=100a1.channels.c1.byteCapacityBufferPercentage=20a1.channels.c1.byteCapacity=800000  #a1.sinks.k1.type=file_roll#注意目录得是虚拟机上的a1.sinks.k1.sink.directory=/tmp/a  # a1.sources.r1.channels=c1a1.sinks.k1.channel=c1#运行命令bin/flume-ng agent --conf conf --name a1 --conf-file conf/fulme-source.conf -Dflume.root.logger=INFO,console

3.结果

阅读全文

0 0