nginx分析日志
来源：互联网 发布：1024邀请码淘宝 编辑：程序博客网 时间：2024/05/18 03:52
#!/usr/bin/env python# -*- coding: utf-8 -*-import timeimport reimport sysimport ip_location#定义一个时间类，可以选取要分析的时间段,如果没有指定时间段，则分析全部logclass TimeParser(object): def __init__(self, re_time, str_time, period): self.__re_time = re.compile(re_time) self.__str_time = str_time self.__period = period def __get(self, line): t = re.search(self.__re_time, line).group(0) return time.mktime(time.strptime(t, self.__str_time)) def inPeriod(self, line): t = self.__get(line) return (t > time.mktime(time.strptime(self.__period[0], self.__str_time)) and t < time.mktime(time.strptime(self.__period[1], self.__str_time)))class ParseLog(object): def __init__(self, file, re_time, str_time, period): #print type(file) self.ip_dict = {} self.url_dict = {} BUF_SIZE = 2048 bigfile = open(file, 'r') try: tmp_lines = bigfile.readlines(BUF_SIZE) while tmp_lines: for eachline in tmp_lines: try: self.domain, self.parsetime, self.suffix = eachline.split("-") except: self.domain = eachline.split(".")[0] self.parsetime = "unknown time" tmp_lines = bigfile.readlines(BUF_SIZE) finally: bigfile.close() #定义一个函数，用来统计数量和总流量，并存入到相应字典中 def Count(self): #用TimeParser实例化CountTime CountTime = TimeParser(re_time, str_time, period) self.total_traffic = [] """ 以下for循环分析每一行，如果这一行不包含时间，就跳过，如果包含时间信息，且在所分析时间段内， 则统计ip和traffic，没有http_refer信息的行只记录ip，然后跳过！ """ with open(file) as f: for i, line in enumerate(f): try: if CountTime.inPeriod(line): ip = line.split()[0] try: traffic = int(re.findall(r'\d{3}\ [^0]\d+', line)[0].split()[1]) except: continue try: url = re.findall(r'GET\ .*\.*\ ', line)[0].split()[1] except IndexError: url = "unknown" else: continue except AttributeError: continue self.ip_dict.setdefault(ip, {'number': 0, 'traffic': 0})['number'] += 1 self.ip_dict.setdefault(ip, {'number': 0, 'traffic': 0})['traffic'] += int(traffic) self.url_dict.setdefault(url, 0) self.url_dict[url] += int(traffic) if not i % 1000000: print "have processed " + str(i) + " lines !" #统计总流量 self.total_traffic.append(int(traffic)) total = sum(self.total_traffic) #打印总流量大小 print "******************************************************************" print self.domain + " all the traffic in " + self.parsetime + " is below:" print "total_traffic: %s " % str(total / 1024 / 1024) + "MB" """定义两个字典，分别存储ip的数量和流量信息""" def TopIp(self, number): self.Count() TopNumberIp = {} TopTrafficIp = {} #对字典赋值 for ip in self.ip_dict.keys(): TopNumberIp[ip] = self.ip_dict[ip]['number'] TopTrafficIp[ip] = self.ip_dict[ip]['traffic'] #按值从大到小的顺序排序键 SortIpNo = sorted(TopNumberIp.items(), key=lambda e: e[1], reverse=True) #print SortIpNo[0][1] SortIpTraffic = sorted(TopTrafficIp.items(), key=lambda e: e[1], reverse=True) #输出连接数top 100 ip的相关信息到文件TopIpNo.txt中 reload(sys) sys.setdefaultencoding('utf-8') ipno = open('D:\\TopIpNo.txt', 'w+') ipno.write(u"ip地址\t\t\t访问次数\t\t国家/区域/城市\t\t\t\t\t\t\t\t\t\t\t\t\t\t运营商\n") ipno.write( "-------------------------------------------------------------------------------------------------\n") for i in range(number): try: ipno.write(SortIpNo[i][0] + "\t\t" + str(SortIpNo[i][1]) + "\t\t\t" + ip_location.ip_location( SortIpNo[i][0]) + "\n") except: continue ipno.write( "-------------------------------------------------------------------------------------------------\n") ipno.close() #输出流量top 100 ip的相关信息到文件iptraffic.txt中 iptr = open('D:\\iptraffic.txt', 'w+') iptr.write(u"ip地址".ljust(15) + u"总流量(MB)".ljust(15) + u"国家/区域/城市".ljust(40) + u"运营商" + "\n") iptr.write( "-------------------------------------------------------------------------------------------------\n") for i in range(number): try: iptr.write(SortIpTraffic[i][0] .ljust(20) + str(SortIpTraffic[i][1] / 1024 / 1024)) #记入地理信息 iptr.write("\t\t\t" + ip_location.ip_location(SortIpTraffic[i][0]) + "\n") except: continue iptr.write( "-------------------------------------------------------------------------------------------------\n") iptr.close() def TopUrl(self, number): SortUrlTraffic = sorted(self.url_dict.items(), key=lambda e: e[1], reverse=True) #输出流量top 100 url相关信息到urltraffic.txt文件中 urtr = open('D:\\urltraffic.txt', 'w+') urtr.write("Filename".ljust(100) + u"TotalTraffic(MB)" + "\n") urtr.write("-----------------------------------------------------------------------------------------\n") for i in range(number): try: urtr.write(SortUrlTraffic[i][0].ljust(100) + ":" + str(SortUrlTraffic[i][1] / 1024 / 1024) + "\n") except: continue urtr.write("-----------------------------------------------------------------------------------------\n") urtr.close()#时间的正则和格式，一般不需要更改re_time = '\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2}'str_time = '%d/%b/%Y:%H:%M:%S'#定义分析的时间段period = ("16/Nov/2000:16:00:00", "16/Nov/2015:17:00:00")#定义输出top numbernumber = 100if __name__ == '__main__': if len(sys.argv) < 2: print 'no logfile specified!' print "Usage: python logParser.py filename" time.sleep(2) sys.exit() else: file = sys.argv[1] lp = ParseLog(file, re_time, str_time, period) startime = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) print "Start to parse the " + file + " struggling!at %s please wait patiently!" % startime print print "******************************************************************" lp.TopIp(number) lp.TopUrl(number) endtime = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) print "The log had parse end at %s !!!" % endtime转自群里一个好友的分享
- awk分析nginx日志
- nginx错误日志分析
- Linux Nginx日志分析
- awstats分析nginx日志
- nginx 日志分析
- nginx分析日志
- Nginx 日志分析
- nginx日志模块分析
- nginx日志分析
- Awstats分析Nginx日志
- nginx日志错误分析
- awstats分析nginx日志
- Awstats分析nginx日志
- goaccess分析nginx日志
- nginx日志分析工具
- Nginx日志分析
- ELK -分析nginx 日志
- nginx日志分析
- MATLAB工具箱大全
- ip地址库分析
- 马云经典录语我没事都在看，很有激情很有力量
- 阿里巴巴2013实习生5.5号（测试开发）笔试！
- 2013蓝桥杯Java本科预赛B组（3）
- nginx分析日志
- 最短路 hdu 2544 水题
- woj 1009 最短路径 dijkstra算法
- linux 下创建文件的方法
- 在特定规律的数组中查找某元素
- ubuntu 安装编译工具
- const总结
- VSTS test controller and agent.
- Inside COM读书笔记------QueryInterface接口
