python 日志封装

日志功能描述：

写python项目时，需要用到日志类，需求为：日志信息可配置，提供几种类型不同的配置，并且日志既可以写到文本也可以写到数据库中。
实现时日志类直接使用python的logging，配置信息写到配置文件logging_data.conf，并使用logging.config.fileConfig(log_config_path)加载配置。写日志到数据库参考了log4mongo-1.6.0.tar.gz的写法，地址在：https://pypi.python.org/pypi/log4mongo/ ，同时每当在数据库写日志时，同时需要插入一些额外信息，比如：projectId runningId algorithmId，所以使用了python的logging.LoggerAdapter把额外信息添加进去。

以下是编写的配置和代码：

配置文件：logging_data.conf

[loggers]keys=root,input,output,computer[handlers]keys=consoleHandler,inputfileHandler,outfileHandler,computerfileHandler,mysqlHandler[formatters]keys=fmt[logger_root]level=DEBUGhandlers=consoleHandler[logger_input]level=DEBUGqualname=inputhandlers=consoleHandler,inputfileHandler,mysqlHandlerpropagate=0[logger_output]level=DEBUGqualname=outputhandlers=consoleHandler,outfileHandler,mysqlHandlerpropagate=0[logger_computer]level=DEBUGqualname=computerhandlers=consoleHandler,computerfileHandler,mysqlHandlerpropagate=0[handler_consoleHandler]class=StreamHandlerlevel=DEBUGformatter=fmtargs=(sys.stdout,)[handler_inputfileHandler]class=logging.handlers.RotatingFileHandlerlevel=DEBUGformatter=fmtargs=('../../logs/input.log','a',20000,5,)[handler_outfileHandler]class=logging.handlers.RotatingFileHandlerlevel=DEBUGformatter=fmtargs=('../../logs/out.log','a',20000,5,)[handler_computerfileHandler]class=logging.handlers.RotatingFileHandlerlevel=DEBUGformatter=fmtargs=('../../logs/computer.log','a',20000,5,)[handler_mysqlHandler]class=MysqlHandler.MysqlHandler#level=WARNINGlevel=DEBUGargs=("10.17.87.226","root","mysql123","aiadm")[formatter_fmt]format=%(asctime)s - %(filename)s:%(lineno)d - %(levelname)s - %(name)s - %(message)sdatefmt=[%Y-%m-%d %H:%M:%S]

从配置中可知：有三种日志配置，分别是input,output,computer,而写日志到数据库则使用了自己编写的MysqlHandler.MysqlHandler

日志包装类 logwrapper.py

#!/usr/bin/env python#coding=UTF-8import loggingimport logging.configimport os,systry:    import thread    import threadingexcept ImportError:    thread = Noneif thread:    _lock = threading.RLock()else:    _lock = Nonedef _acquireLock():    """    Acquire the module-level lock for serializing access to shared data.    This should be released with _releaseLock().    """    if _lock:        _lock.acquire()def _releaseLock():    """    Release the module-level lock acquired by calling _acquireLock().    """    if _lock:        _lock.release()class LoggerAdapter(logging.LoggerAdapter):    def process(self,msg,kwargs):        try:            kwargs["extra"] = dict(kwargs["extra"],**self.extra)        except:            kwargs["extra"] = self.extra        return msg,kwargs    def setExtra(self,extra):        self.extra = extraclass logwrapper:    def __init__(self):        cur_dir =  os.path.dirname(os.path.abspath(__file__))        log_config_path = cur_dir + "/logging_data.conf"        sys.path.append(cur_dir)        logging.config.fileConfig(log_config_path)        self.logger_dict = {}    def getLogger(self,name,extra = {}):        rv = None        _acquireLock()        try:            if name in self.logger_dict:                rv = self.logger_dict[name]            else:                logger = logging.getLogger(name)                rv = LoggerAdapter(logger, extra)                self.logger_dict[name] = rv        finally:            _releaseLock()        return rvlog_manager = logwrapper()def getLogger(name=None):        return log_manager.getLogger(name)

写日志到数据库的类MysqlHander.py

import loggingimport MySQLdbimport datetimeimport syssys.path.append("..")import data_service.mysql_context.mySQLWrap as mySQLWrap_mysql_obj = Noneclass MysqlHandler(logging.Handler):    def __init__(self,host,user,passwd,db,charset="utf8",timeout=10,level=logging.NOTSET,reuse=True):        logging.Handler.__init__(self,level)        self.host = host        self.user = user        self.passwd = passwd        self.db = db        self.charset = charset        self.timeout = timeout        self.reuse = reuse        self.mysql_obj  = None        self.connect_state = False        self._connect()    def _connect(self):        global _mysql_obj        if self.reuse and _mysql_obj:            self.mysql_obj = _mysql_obj        else:            self.mysql_obj = mySQLWrap.MySQLWrap()            ret = self.mysql_obj.connectDatabase(host=self.host,user=self.user,passwd=self.passwd,db=self.db,charset=self.charset,timeout=self.timeout)            if ret == False:                _mysql_obj = None                self.connect_state = False                self.printMysqlError(self.mysql_obj.getErrorStr(),"MysqlHandler mysql connect error")            else:                _mysql_obj = self.mysql_obj                self.connect_state = True    def emit(self,record):        if self.connect_state == False:            return        CRITICAL = 50        FATAL = CRITICAL        ERROR = 40        WARNING = 30        WARN = WARNING        INFO = 20        DEBUG = 10        NOTSET = 0        _level_names = {            'CRITICAL': CRITICAL,            'ERROR': ERROR,            'WARN': WARNING,            'WARNING': WARNING,            'INFO': INFO,            'DEBUG': DEBUG,            'NOTSET': NOTSET,        }        level = _level_names[record.levelname]        log_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")        log_desc = record.getMessage()        keys = set(record.__dict__)        project_id = "NULL"        running_id = "NULL"        algorithm_id = "NULL"        ext1 = ext2 = ext3 = ext4 = ext5 = ext6 = ext7 = ext8 = ""        if keys:            for key in keys:                if key == "project_id":                    project_id = record.__dict__["project_id"]                elif key == "running_id":                    running_id = record.__dict__["running_id"]                elif key == "Algorithm_id":                    algorithm_id = record.__dict__["Algorithm_id"]                elif key == "ext1":                    ext1 = record.__dict__["ext1"]                elif key == "ext2":                    ext2 = record.__dict__["ext2"]                elif key == "ext3":                    ext3 = record.__dict__["ext3"]                elif key == "ext4":                    ext4 = record.__dict__["ext4"]                elif key == "ext5":                    ext5 = record.__dict__["ext5"]                elif key == "ext6":                    ext6 = record.__dict__["ext6"]                elif key == "ext7":                    ext7 = record.__dict__["ext7"]                elif key == "ext8":                    ext8 = record.__dict__["ext8"]        sql = '''INSERT INTO AI_ALGORITHM_LOGS(running_id,                    project_id,Algorithm_id,log_time,level,log_desc,ext1,ext2,ext3,ext4,ext5,ext6,ext7,ext8)                    VALUES('%s','%s','%s','%s','%d',"%s",'%s','%s','%s','%s','%s','%s','%s','%s')'''\                    %(str(running_id),str(project_id),str(algorithm_id),log_time,level,log_desc,ext1,ext2,ext3,ext4,ext5,ext6,ext7,ext8)        #print sql        ret = self.mysql_obj.exeSQLcmd(sql)        if ret == False:            self.printMysqlError(self.mysql_obj.getErrorStr(),"MysqlHandler insert mysql log error")    def printMysqlError(self,str,msg):        print "%s %s" %(msg,str)    def close(self):        self.mysql_obj.closeDatabase()    def __del__(self):        self.mysql_obj.closeDatabase()    def __exit__(self, exc_type, exc_val, exc_tb):        self.mysql_obj.closeDatabase()

其中projectId runningId algorithmId是每条日志都必须到数据库插入的数据，而ext1 -> ext8则是可能插入的一些额外信息。mySQLWrap.MySQLWrap是自己封装的使用mysqldb操作数据库的一个类。

测试使用logtest.py

#!/usr/bin/env python#coding=UTF-8import logwrapperlogwrapper = logwrapper.getLogger("input")logwrapper.setExtra({"project_id":2,"running_id":0,"Algorithm_id":1})#handler = MysqlHandler.MysqlHandler(host="10.17.87.226",user="root",passwd="mysql123",db="aiadm")#logger.addHandler(handler)logger.info("mysql logger %d,%s",1,"hello",extra={"ext3":"extrss","ext4":"extss444"})logger.info("mysql logger %d,%s",1,"hello",extra={"ext1":"extrss","ext2":"extss444"})logger.info("test input info logger %d,%s",1,"hello")logger.warning("test input warning logger %d,%s",1,"hello")logger.error("test input logger %d,%s",1,"hello")logger.critical("test input logger %d,%s",1,"hello")try:    1/0except:    logger.exception("except")logger = logwrapper.getLogger("out")logger.debug("test out debug logger %d,%s",1,"hello")logger.info("test out info logger %d,%s",1,"hello")