从Spark Streaming 收集应用程序日志到Flume agent的配置方法

从Spark Streaming 收集应用程序日志到Flume agent的配置方法

由于Spark 本身也是log4j收集日志，所以我们在应用程序里再配置一个log4j。先说一下spark Streaming 启动方法，在spark 启动的时候添加以下两个参数：--conf "spark.executor.extraJavaOptions=-Dlog4j.configuration=log4j.properties". 比如：

spark-submit --class com.juniorchina.modeling.entry.spark.streaming.Main \  --num-executors 5 \  --executor-cores 8 \  --master yarn \  --deploy-mode cluster \  --files "log4j-executor.properties" \  --conf "spark.executor.extraJavaOptions=-Dlog4j.configuration=log4j-executor.properties" \  user_tag-1.0-SNAPSHOT-jar-with-dependencies.jar

log4j-executor.properties配置

# Define some default values that can be overridden by system properties.## For testing, it may also be convenient to specify# -Dflume.root.logger=DEBUG,console when launching flume.# Set everything to be logged to the consolelog4j.rootCategory=INFO,consolelog4j.logger.UserModelJob=INFO,console,flume# don't log to rootCategorylog4j.additivity.UserModelJob=false# info log filelog4j.appender.file=org.apache.log4j.DailyRollingFileAppenderlog4j.appender.file.Encoding=UTF8log4j.appender.file.Threshold=INFOlog4j.appender.file.File=logs/flume.loglog4j.appender.file.DatePattern='_'yyyyMMdd'.log'log4j.appender.file.layout=org.apache.log4j.PatternLayoutlog4j.appender.file.layout.ConversionPattern=[%p] [%d{yy/MM/dd HH:mm:ss}] [%c{1}]: %m%n# LOGFILE# Stock log4j rolling file appender# Default log rotation configurationlog4j.appender.LOGFILE=org.apache.log4j.RollingFileAppenderlog4j.appender.LOGFILE.MaxFileSize=10KBlog4j.appender.LOGFILE.MaxBackupIndex=10log4j.appender.LOGFILE.File=${flume.log.dir}/${flume.log.file}log4j.appender.LOGFILE.layout=org.apache.log4j.PatternLayoutlog4j.appender.LOGFILE.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS}|%p|%l|%t|%c|%m%n# Warning: If you enable the following appender it will fill up your disk if you don't have a cleanup job!# This uses the updated rolling file appender from log4j-extras that supports a reliable time-based rolling policy.# See http://logging.apache.org/log4j/companions/extras/apidocs/org/apache/log4j/rolling/TimeBasedRollingPolicy.html# Add "DAILY" to flume.root.logger above if you want to use thislog4j.appender.DAILY=org.apache.log4j.rolling.RollingFileAppenderlog4j.appender.DAILY.rollingPolicy=org.apache.log4j.rolling.TimeBasedRollingPolicylog4j.appender.DAILY.rollingPolicy.ActiveFileName=${flume.log.dir}/${flume.log.file}.loglog4j.appender.DAILY.rollingPolicy.FileNamePattern=${flume.log.dir}/${flume.log.file}.%d{yyyyMMddHH}.loglog4j.appender.DAILY.layout=org.apache.log4j.PatternLayoutlog4j.appender.DAILY.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss},%c,%m,(%C.%M:%L)%n# console# Add "console" to flume.root.logger above if you want to use thislog4j.appender.console=org.apache.log4j.ConsoleAppenderlog4j.appender.console.target=System.outlog4j.appender.console.layout=org.apache.log4j.PatternLayoutlog4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss},(%F),%c,%m%n# flumelog4j.appender.flume=org.apache.flume.clients.log4jappender.Log4jAppenderlog4j.appender.flume.Hostname=192.168.5.151log4j.appender.flume.Port=41414log4j.appender.flume.AppName=usermodelinglog4j.appender.flume.UnsafeMode=falselog4j.appender.flume.layout=org.apache.log4j.PatternLayoutlog4j.appender.flume.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss},(%F),%c,%m%n# Settings to quiet third party logs that are too verboselog4j.logger.org.eclipse.jetty=WARNlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFOlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO

log4jappender.Log4jAppender

对log4jappender.Log4jAppender做了一些修改，在header中添加app.name，app.event.type,app.source.host参数。核心代码：

 @Override    public synchronized void append(LoggingEvent event) throws FlumeException {        //If rpcClient is null, it means either this appender object was never        //setup by setting hostname and port and then calling activateOptions        //or this appender object was closed by calling close(), so we throw an        //exception to show the appender is no longer accessible.        boolean ready = true;        if (rpcClient == null) {            String errorMsg = "Cannot Append to Appender! Appender either closed or" +                    " not setup correctly!";            LogLog.error(errorMsg);            // start a new thread try            if (unsafeMode) {                reconnect();                if(rpcClient == null || !rpcClient.isActive()){                    ready = false;                }            }else {                throw new FlumeException(errorMsg);            }        }        if(unsafeMode && !ready){            return;        }        if (!rpcClient.isActive()) {            reconnect();        }        //Client created first time append is called.        Map<String, String> hdrs = new HashMap<String, String>();        //hdrs.put(Log4jAvroHeaders.LOGGER_NAME.toString(), event.getLoggerName());        hdrs.put(Log4jAvroHeaders.TIMESTAMP.toString(),                String.valueOf(event.timeStamp));        //To get the level back simply use        //LoggerEvent.toLevel(hdrs.get(Integer.parseInt(        //Log4jAvroHeaders.LOG_LEVEL.toString()))        hdrs.put(Log4jAvroHeaders.LOG_LEVEL.toString(),                String.valueOf(event.getLevel().toInt()));        hdrs.put(Log4jAvroHeaders.APP_NAME.toString(), appName);        hdrs.put(Log4jAvroHeaders.APP_Event_TYPE.toString(), event.getLoggerName());        try {            String hostIp = null;            Enumeration<NetworkInterface> networkInterfaces = NetworkInterface.getNetworkInterfaces();            while (networkInterfaces.hasMoreElements()) {                NetworkInterface ni = (NetworkInterface) networkInterfaces.nextElement();                Enumeration<InetAddress> nias = ni.getInetAddresses();                while (nias.hasMoreElements()) {                    InetAddress ia = (InetAddress) nias.nextElement();                    if (!ia.isLinkLocalAddress()                            && !ia.isLoopbackAddress()                            && ia instanceof Inet4Address) {                        hostIp = ia.getHostAddress();                    }                }            }            hdrs.put(Log4jAvroHeaders.APP_SOURCE_HOST.toString(), hostIp + "");        } catch (SocketException e) {            String msg = "Can't get localhost IP";            LogLog.error(msg);            if (unsafeMode) {                return;            }            throw new FlumeException(msg + " Exception follows.", e);        }        Event flumeEvent;        Object message = event.getMessage();        if (message instanceof GenericRecord) {            GenericRecord record = (GenericRecord) message;            populateAvroHeaders(hdrs, record.getSchema(), message);            flumeEvent = EventBuilder.withBody(serialize(record, record.getSchema()), hdrs);        } else if (message instanceof SpecificRecord || avroReflectionEnabled) {            Schema schema = ReflectData.get().getSchema(message.getClass());            populateAvroHeaders(hdrs, schema, message);            flumeEvent = EventBuilder.withBody(serialize(message, schema), hdrs);        } else {            hdrs.put(Log4jAvroHeaders.MESSAGE_ENCODING.toString(), "UTF8");            String msg = layout != null ? layout.format(event) : message.toString();            flumeEvent = EventBuilder.withBody(msg, Charset.forName("UTF8"), hdrs);        }        try {            rpcClient.append(flumeEvent);        } catch (Exception e) {            String msg = "Flume append() failed.";            LogLog.error(msg);            if (unsafeMode) {                return;            }            throw new FlumeException(msg + " Exception follows.", e);        }    }

可以去github clone flume-log4j-appender https://github.com/kntao/flume/tree/flume-1.5/flume-ng-clients/flume-ng-log4jappender