日志文件的处理和分析

package com.tudou.thread;import java.io.BufferedReader;import java.io.File;import java.io.FileReader;import java.io.FileWriter;import java.io.IOException;import java.lang.Thread.State;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;public class ReadProcessedFile {private static BufferedReader bReader = null;private static String content = null;private static String splitInfo[] = null;private static final int COUNT = 1;//private static  int test_int = 0;/* * 读取文件，取出所有的INFO日志存于另一个文件中 */public static void readFile(String log,String log_info) throws IOException {File inputfile = new File(log);bReader = new BufferedReader(new FileReader(inputfile));while((content = bReader.readLine()) != null) {System.out.println(content);if(content.startsWith("2014-12-09")){splitInfo = content.split(" ");if(splitInfo[3].equals("INFO")) {writeFile(content,log_info);}}}bReader.close();}/* * 内容写入文件 */public static void writeFile(String content,String log_info) throws IOException {File outputFile = new File(log_info);if(!outputFile.exists()) {outputFile.createNewFile();}FileWriter fWriter = new FileWriter(outputFile.getAbsoluteFile(),true);fWriter.write(content+"\n");fWriter.close();}/* * map对象写入文件 */public static void writeObjectToFile(Map<String,Integer> map,String final_result) throws IOException{File file = new File(final_result);if(!file.exists()){file.createNewFile();}FileWriter fWriter = new FileWriter(file.getAbsoluteFile(),true);StringBuilder param = new StringBuilder();for(String key : map.keySet()) {param.append(key+"="+map.get(key)+",");}fWriter.write(param.toString()+"\n\n");fWriter.close();//System.out.println("成功写入文件");}/* * 将INFO日志的content部分存到另一个文件中； */public static void readProcessedFile(String log_info,String info_content) throws IOException {File file = new File(log_info);bReader = new BufferedReader(new FileReader(file));while((content = bReader.readLine()) != null){System.out.println(content);splitInfo = content.split(" - ");if(splitInfo.length > 1){writeFile(splitInfo[1], info_content);}}bReader.close();}/* * 统计各个s3_app类型的访问量 */public static void readLogContent(String info_content,String final_result) throws IOException {File file = new File(info_content);bReader = new BufferedReader(new FileReader(file));String s3_app = null;Map<String, Integer> map = null;while((content = bReader.readLine()) != null){if(content.trim().startsWith("s3__app=")){splitInfo = content.split(" ");s3_app = splitInfo[0].substring(9,splitInfo[0].length()-1);if(map == null) {map = new HashMap<String, Integer>();map.put(s3_app, COUNT) ;}else {int state = 0;for(String key : map.keySet()){state ++;if(s3_app.equals(key)){map.put(s3_app, map.get(s3_app)+1);break;}if(state == map.size()){map.put(s3_app, COUNT);}}}}}bReader.close();writeObjectToFile(map,final_result);System.out.println("Each s3_app Model的访问量->" + map);}/* * s3_app访问 Each app Model的访问量 */public static void getVisitedInfoByLogContent(String info_content,String final_result) throws IOException{File file = new File(info_content);bReader = new BufferedReader(new FileReader(file));Map<String, Integer> map = null;String visitInfo = null;//String s3_app = null;while((content = bReader.readLine()) != null){if(content.trim().startsWith("s3__app=")){splitInfo = content.split(" ");visitInfo = splitInfo[0].substring(9,splitInfo[0].length()-1).concat("-"+splitInfo[5].substring(5,splitInfo[5].length()-1));if(map == null) {map = new HashMap<String ,Integer>();map.put(visitInfo, COUNT);}else {int state = 0;for(String key : map.keySet()){state ++;if(visitInfo.equals(key)){map.put(visitInfo, map.get(visitInfo)+1);break;}if(state == map.size()){map.put(visitInfo,COUNT);}}}}}bReader.close();System.out.println("s3_app each app Model的访问量->"+map);writeObjectToFile(map,final_result);}/* * each app param 的 访问量 */public static void getEachParamInfoByLogContent(String info_content,String final_result) throws IOException {File file = new File(info_content);bReader = new BufferedReader(new FileReader(file));//List<String> paramList = new ArrayList<String>() ;String param[] = null;//String eachParam[] = null;String paramArray[] = null;String paramInfo[] = null;//String splitParamInfo[] = null;Map<String,String> paramValue = new HashMap<String,String>();Map<String,Integer> statisticsParam = new HashMap<String,Integer>();StringBuilder totalParam = new StringBuilder();while((content = bReader.readLine()) != null){//test_int ++;if(content.trim().startsWith("s3__app=")){splitInfo = content.split(" FakeSolrParam ");param = splitInfo[1].split(",");int state = -1;int conLocation = 0;int memLocation = 0;for(String eachParam : param) {//System.out.println(eachParam);state ++;if(eachParam.trim().startsWith("[app")){totalParam.append(eachParam.trim()+",");}if(eachParam.trim().startsWith("sortAsc")){totalParam.append(eachParam.trim()+",");continue;}if(eachParam.trim().startsWith("sortDesc")){totalParam.append(eachParam.trim()+",");continue;}if(eachParam.trim().startsWith("conditionFields")){//totalParam.append(eachParam+",");conLocation = state;//System.out.println("conLocation="+conLocation);continue;}if(eachParam.trim().startsWith("memcacheTime")){memLocation = state;//System.out.println("memLocation="+memLocation);}}for(int i=conLocation;i<memLocation;i++){totalParam.append(param[i].trim()+"-");}paramArray = totalParam.toString().split(",");for(String param1:paramArray) {paramInfo = param1.split("=");paramValue.put(paramInfo[0], paramInfo[1]);}//System.out.println(test_int);//System.out.println(paramValue);if(paramValue.get("sortAsc") != null && !paramValue.get("sortAsc").trim().equals("[]")){String key = paramValue.get("[app")+"-"+"sortAsc"+"-"+paramValue.get("sortAsc").substring(1, paramValue.get("sortAsc").length()-1);if(statisticsParam.size()==0){statisticsParam.put(key, COUNT);}else {int state1 = 0;for(String statisticsKey:statisticsParam.keySet()) {state1 ++;if(key.equals(statisticsKey)){statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);break;}if(statisticsParam.size() == state1){statisticsParam.put(key, COUNT);}}}}if(paramValue.get("sortDesc") != null && !paramValue.get("sortDesc").trim().equals("[]")){String key = paramValue.get("[app")+"-"+"sortDesc"+"-"+paramValue.get("sortDesc").substring(1,paramValue.get("sortDesc").length()-1);if(statisticsParam.size()==0){statisticsParam.put(key, COUNT);}else {int state2 = 0;for(String statisticsKey:statisticsParam.keySet()) {state2 ++;if(key.equals(statisticsKey)){statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);break;}if(statisticsParam.size() == state2){statisticsParam.put(key, COUNT);}}}}if(paramValue.get("conditionFields") != null && !paramValue.get("conditionFields").equals("[]")){String conditionParam[] = paramValue.get("conditionFields").split("-");if(conditionParam.length != 1){int flag = 0;for(String conditionValue : conditionParam){flag ++;String key = null;if(flag == 1) {key = paramValue.get("[app")+"-"+"fq"+"-"+conditionValue.substring(1,conditionValue.length());}else if(flag == conditionParam.length){key = paramValue.get("[app")+"-"+"fq"+"-"+conditionValue.substring(0,conditionValue.length()-1);}else{key = paramValue.get("[app")+"-"+"fq"+"-"+conditionValue.substring(0,conditionValue.length());}if(!key.equalsIgnoreCase(paramValue.get("[app")+"-fq-null") && !key.equalsIgnoreCase(paramValue.get("[app")+"-fq-")){if(statisticsParam.size() == 0){statisticsParam.put(key, COUNT);}else {int state3 = 0;for(String statisticsKey:statisticsParam.keySet()) {state3 ++;if(key.equals(statisticsKey)){statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);break;}if(statisticsParam.size() == state3){statisticsParam.put(key, COUNT);}}}}}}else{String key = paramValue.get("[app")+"-"+"fq"+"-"+paramValue.get("conditionFields").substring(1,paramValue.get("conditionFields").length()-2);if(!key.equalsIgnoreCase(paramValue.get("[app")+"-fq-null") && !key.equalsIgnoreCase(paramValue.get("[app")+"-fq-")){if(statisticsParam.size()==0){statisticsParam.put(key, COUNT);}else {int state4 = 0;for(String statisticsKey:statisticsParam.keySet()) {state4 ++;if(key.equals(statisticsKey)){statisticsParam.put(statisticsKey, statisticsParam.get(statisticsKey)+1);break;}if(statisticsParam.size() == state4){statisticsParam.put(key, COUNT);}}}}}}paramValue.clear();totalParam.delete(0, totalParam.length());}}bReader.close();System.out.println("each app param 的 访问量->"+statisticsParam);writeObjectToFile(statisticsParam,final_result);}

public static void main(String args[]) throws IOException {readFile("D:\\s3_interface.log.2014-12-09", "D:\\log_Info.txt");readProcessedFile("D:\\log_Info.txt","D:\\info_content.txt");readLogContent("D:\\info_content.txt","D:\\final_result.txt");getVisitedInfoByLogContent("D:\\info_content.txt","D:\\final_result.txt");getEachParamInfoByLogContent("D:\\info_content.txt","D:\\final_result.txt");}}