Hadoop 分析Apache日志记录 URLlog日志分析

来源：互联网发布：java 14位时间戳校验编辑：程序博客网时间：2024/04/29 08:33

Hadoop 分析Apache日志记录 URLlog日志分析：

1、GET方式的URL出现的次数

2、PUT方式的URL出现的次数

数据文件

[root@master IMFdatatest]#hadoop dfs -cat /library/URLLog.txt
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.

16/02/16 07:23:53 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
127.0.0.1 - - [03/Jul/2015:23:36:38 +0800] "GET /course/detail/3.htm HTTP/1.0" 200 38435 0.038
182.131.89.195 - - [03/Jul/2015:23:37:43 +0800] "GET / HTTP/1.0" 301 - 0.000
127.0.0.1 - - [03/Jul/2015:23:38:27 +0800] "POST /service/notes/addViewTimes_23.htm HTTP/1.0" 200 2 0.003
127.0.0.1 - - [03/Jul/2015:23:39:03 +0800] "GET /html/notes/20140617/779.html HTTP/1.0" 200 69539 0.046
127.0.0.1 - - [03/Jul/2015:23:43:00 +0800] "GET /html/notes/20140318/24.html HTTP/1.0" 200 67171 0.049
127.0.0.1 - - [03/Jul/2015:23:43:59 +0800] "POST /service/notes/addViewTimes_779.htm HTTP/1.0" 200 1 0.003
127.0.0.1 - - [03/Jul/2015:23:45:51 +0800] "GET / HTTP/1.0" 200 70044 0.060
127.0.0.1 - - [03/Jul/2015:23:46:17 +0800] "GET /course/list/73.htm HTTP/1.0" 200 12125 0.010
127.0.0.1 - - [03/Jul/2015:23:46:58 +0800] "GET /html/notes/20140609/542.html HTTP/1.0" 200 94971 0.077
127.0.0.1 - - [03/Jul/2015:23:48:31 +0800] "POST /service/notes/addViewTimes_24.htm HTTP/1.0" 200 2 0.003
127.0.0.1 - - [03/Jul/2015:23:48:34 +0800] "POST /service/notes/addViewTimes_542.htm HTTP/1.0" 200 2 0.003
127.0.0.1 - - [03/Jul/2015:23:49:31 +0800] "GET /notes/index-top-3.htm HTTP/1.0" 200 53494 0.041
127.0.0.1 - - [03/Jul/2015:23:50:55 +0800] "GET /html/notes/20140609/544.html HTTP/1.0" 200 183694 0.076
127.0.0.1 - - [03/Jul/2015:23:53:32 +0800] "POST /service/notes/addViewTimes_544.htm HTTP/1.0" 200 2 0.004
127.0.0.1 - - [03/Jul/2015:23:54:53 +0800] "GET /html/notes/20140620/900.html HTTP/1.0" 200 151770 0.054
127.0.0.1 - - [03/Jul/2015:23:57:42 +0800] "GET /html/notes/20140620/872.html HTTP/1.0" 200 52373 0.034
127.0.0.1 - - [03/Jul/2015:23:58:17 +0800] "POST /service/notes/addViewTimes_900.htm HTTP/1.0" 200 2 0.003
127.0.0.1 - - [03/Jul/2015:23:58:51 +0800] "GET / HTTP/1.0" 200 70044 0.057[root@master IMFdatatest]#

运行结果

[root@master IMFdatatest]#hadoop dfs -cat /library/outputURLLog/part-r-00000
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.

16/02/16 07:13:48 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
GET /   3
GET /course/detail/3.htm        1
GET /course/list/73.htm 1
GET /html/notes/20140318/24.html        1
GET /html/notes/20140609/542.html       1
GET /html/notes/20140609/544.html       1
GET /html/notes/20140617/779.html       1
GET /html/notes/20140620/872.html       1
GET /html/notes/20140620/900.html       1
GET /notes/index-top-3.htm      1
POST /service/notes/addViewTimes_23.htm 1
POST /service/notes/addViewTimes_24.htm 1
POST /service/notes/addViewTimes_542.htm        1
POST /service/notes/addViewTimes_544.htm        1
POST /service/notes/addViewTimes_779.htm        1
POST /service/notes/addViewTimes_900.htm        1

源代码

package com.dtspark.hadoop.hellomapreduce;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.StringTokenizer;

public class URLLog {

public static class DataMapper
     extends Mapper<LongWritable, Text, Text, LongWritable>{

   private LongWritable resultValue = new LongWritable(1);
private Text text =new Text();

   public void map(LongWritable key, Text value, Context context
                   ) throws IOException, InterruptedException {

    System.out.println("Map Methond Invoked!!!");
    String line =value.toString();
    String result = handleLine(line);
    if (result != null && result.length() > 0 ){
     text.set(result);
     context.write(text, resultValue);
    }

}

private String handleLine(String line) {

  StringBuffer buffer = new StringBuffer();
  if(line.length()>0){
   if(line.contains("GET")){
    buffer.append(line.substring(line.indexOf("GET"),line.indexOf("HTTP/1.0")).trim());
   }else if ( line.contains("POST")) {
    buffer.append(line.substring(line.indexOf("POST"),line.indexOf("HTTP/1.0")).trim());

   }

  }

  return buffer.toString();
}

}

public static class DataReducer
extends Reducer<Text,LongWritable,Text, LongWritable> {
private LongWritable totalresultValue = new LongWritable(1);

public void reduce(Text key, Iterable<LongWritable> values,
                      Context context
                      ) throws IOException, InterruptedException {
    System.out.println("Reduce Methond Invoked!!!" );
    int total =0;
    for (LongWritable item : values){
     total += item.get();
    }
    totalresultValue.set(total);
    context.write(key, totalresultValue);

   }

}

public static void main(String[] args) throws Exception {

Configuration conf = new Configuration();

String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length < 2) {
System.err.println("Usage: URLLog <in> [<in>...] <out>");
System.exit(2);
}

Job job = Job.getInstance(conf, "URLLog");
job.setJarByClass(URLLog.class);
job.setMapperClass(DataMapper.class);
job.setReducerClass(DataReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
for (int i = 0; i < otherArgs.length - 1; ++i) {
FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
}
FileOutputFormat.setOutputPath(job,
new Path(otherArgs[otherArgs.length - 1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}

}

0 0