python大数据分析操作系统日志

一 代码

1、大文件切分

import osimport os.pathimport timedef FileSplit(sourceFile, targetFolder):    if not os.path.isfile(sourceFile):        print(sourceFile, ' does not exist.')        return    if not os.path.isdir(targetFolder):        os.mkdir(targetFolder)    tempData = []    number = 1000    fileNum = 1    linesRead = 0        with open(sourceFile, 'r') as srcFile:        dataLine = srcFile.readline().strip()        while dataLine:            for i in range(number):                tempData.append(dataLine)                                dataLine = srcFile.readline()                                if not dataLine:                    break            desFile = os.path.join(targetFolder, sourceFile[0:-4] + str(fileNum) + '.txt')            with open(desFile, 'a+') as f:                f.writelines(tempData)            tempData = []            fileNum = fileNum + 1       if __name__ == '__main__':    #sourceFile = input('Input the source file to split:')    #targetFolder = input('Input the target folder you want to place the split files:')    sourceFile = 'test.txt'    targetFolder = 'test'    FileSplit(sourceFile, targetFolder)

 

2、Mapper代码

import osimport reimport threadingimport timedef Map(sourceFile):    if not os.path.exists(sourceFile):        print(sourceFile, ' does not exist.')        return        pattern = re.compile(r'[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}')    result = {}    with open(sourceFile, 'r') as srcFile:        for dataLine in srcFile:            r = pattern.findall(dataLine)            if r:                t = result.get(r[0], 0)                t += 1                result[r[0]] = t    desFile = sourceFile[0:-4] + '_map.txt'    with open(desFile, 'a+') as fp:        for k, v in result.items():            fp.write(k + ':' + str(v) + '\n')if __name__ == '__main__':    desFolder = 'test'    files = os.listdir(desFolder)        #如果不使用多线程，可以直接这样写    '''for f in files:        Map(desFolder + '\\' + f)'''        #使用多线程    def Main(i):        Map(desFolder + '\\' + files[i])    fileNumber = len(files)    for i in range(fileNumber):        t = threading.Thread(target = Main, args =(i,))        t.start()

 

3.Reducer代码

import osdef Reduce(sourceFolder, targetFile):    if not os.path.isdir(sourceFolder):        print(sourceFolder, ' does not exist.')        return    result = {}    #Deal only with the mapped files    allFiles = [sourceFolder+'\\'+f for f in os.listdir(sourceFolder) if f.endswith('_map.txt')]    for f in allFiles:        with open(f, 'r') as fp:            for line in fp:                line = line.strip()                if not line:                    continue                position = line.index(':')                key = line[0:position]                value = int(line[position + 1:])                result[key] = result.get(key,0) + value    with open(targetFile, 'w') as fp:        for k,v in result.items():            fp.write(k + ':' + str(v) + '\n')if __name__ == '__main__':    Reduce('test', 'test\\result.txt')

 

二 运行结果

依次运行上面3个程序，得到最终结果：

07/10/2013:4634

07/16/2013:51

08/15/2013:3958

07/11/2013:1

10/09/2013:733

12/11/2013:564

02/12/2014:4102

05/14/2014:737