Centos6.5 64位 安装Hadoop2.7.0, MapReduce日志分析, Hive2.1.0, JDBC连接Hive查询 (2)

第二篇 MapReduce日志分析

做日志分析之前, 我觉得要先了解下MapReduce , 网上很多, 你可以搜下, 这位哥们讲的还不错 点击打开链接

日志长这样的:

<?php  if ( ! defined('ROOT_PATH')) exit('No direct script access allowed'); ?>[2016-06-01 00:10:27] POST 218.82.131.157 /user/HealthCenter.php?m=submit uid=14&hash=dd16e3e4d0e8786f13166a4065f24fa0&num=13.0&type=1&time=1464711029 0(OK)[2016-06-01 08:10:27] POST 218.82.131.157 /user/HealthCenter.php?m=submit uid=14&hash=863fbf2535639c16d885a55c78dff665&num=13.0&type=1&time=1464739829 0(OK)[2016-06-01 09:10:28] POST 124.74.69.134 /user/HealthCenter.php?m=submit uid=14&hash=9a310b722e795e2673cdba76bea29b26&num=13.0&type=1&time=1464743429 0(OK)[2016-06-01 09:16:05] GET 124.74.69.134 /index/Main.php?hash=eac57627d3407963dab81da2bb07e378&page_num=1&page_size=10&time=1464743769&uid=8 0(OK)[2016-06-01 10:01:30] GET 124.74.69.134 /index/Main.php?hash=7979353ef669c61f75a5a7e9d39cd646&page_num=1&page_size=10&time=1464746494&uid=8 0(OK)[2016-06-01 10:10:28] POST124.74.69.134 /user/HealthCenter.php?m=submit uid=14&hash=98012832769b5a0e45f036e92032f1ef&num=13.0&type=1&time=1464747029 0(OK)[2016-06-01 10:11:12] GET 124.74.69.134 /index/Main.php?hash=77938b0fdf1b733a9e15d9a2055767d1&page_num=1&page_size=10&time=1464747076&uid=8 0(OK)[2016-06-01 10:48:00] GET 124.74.69.134 /index/Main.php?hash=1a1979e9fdcfca2f17bf1b287f4508aa&page_num=1&page_size=10&time=1464749284&uid=8 0(OK)[2016-06-01 10:48:42] POST 124.74.69.134 /user/Position.php uid=9&address=undefine&latitude=4.9E-324&time=1464749394&type=1&hash=d4aebe936762be3a0420b62a77e37b00&longitude=4.9E-324 0(OK)

分别是: 时间 请求方式 IP 请求地址 参数 返回值

每天产生一个, 分别已Y-m-d.php 方式命名.

达到的目的是: 统计每天 每个接口的请求次数, 以返回结果分组,

编写程序

/** * @ClassName:     LogMapReduce.java * @author         [e-mail] * @version        V1.0  * @Date           2016-7-11 10:20:11 * @Description:   TODO * */package www.com.cn;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;public class LogMapReduce extends Configured implements Tool {public static void main(String[] args) {Configuration conf = new Configuration();try {int res = ToolRunner.run(conf, new LogMapReduce(), args);System.exit(res);} catch (Exception e) {e.printStackTrace();}}@Overridepublic int run(String[] args) throws Exception {Configuration conf = new Configuration();final Job job = Job.getInstance(conf, "LogParaseMapReduce");job.setJarByClass(LogMapReduce.class);FileInputFormat.setInputPaths(job, args[0]);job.setMapperClass(MyMapper.class);job.setReducerClass(MyReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);FileOutputFormat.setOutputPath(job, new Path(args[1]));boolean success = job.waitForCompletion(true);if (success) {System.out.println("process success!");} else {System.out.println("process failed!");}return 0;}    enum Counter{          LINESKIP,      }     static class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable> {private final static IntWritable one = new IntWritable(1);public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {if ("".equals(value)) {return;}String line = value.toString();if (line.length() == 0 || !"[".equals(line.substring(0, 1))) {return;}try {String[] lines = line.split(" ");String data = lines[0].replace("[", "") + "\t";if ("GET".equals(lines[2])) {String url = "";String[] urls = lines[4].split("[?]");String[] params = urls[1].split("[&]");if (params[0].indexOf('m') != -1) {url = urls[0] + "?" + params[0];} else {url = urls[0];}data += url + "\t" + lines[5];} else if ("POST".equals(lines[2])) {data += lines[4] + "\t" + lines[6];}Text out = new Text(data);context.write(out, one);} catch (ArrayIndexOutOfBoundsException e) {context.getCounter(Counter.LINESKIP).increment(1); }}}static class MyReducer extends Reducer<Text, IntWritable, Text, IntWritable> {protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {int count = 0;for (IntWritable v: values) {count = count + 1;}context.write(key, new IntWritable(count));}}}

然后将程序打jar包:LogMapReduce.jar

打包时注意选择 Main class, 这时候选择就不用在调用时指定包目录了

Shell脚本

脚本分为两个, 一个执行脚本, 一个运行执行的脚本

运行脚本 run.sh:

#! /bin/bashd=`date "+%Y-%m-%d %H:%M:%S"`echo "{$d} start..."file=$1;if [ -f ${file} ];then        echo "${file} exists"else        echo "${file} not exists"        exit 0fi#获取文件名fileinfo=(${file//// })filename=${fileinfo[$[${#fileinfo[@]}-1]]}info=(${filename//./ })name=${info[0]}echo "hadoop put file to /api/put/${filename}"hadoop fs -put ${file} /api/put/${filename}echo "call LogMapReduce.jar"hadoop jar /home/hadoop/hadoop-2.7.0/share/hadoop/mapreduce/LogMapReduce.jar /api/put/${filename} /api/out/${name}echo "hive load into api_logs"hive -e "load data inpath '/api/out/${name}/part-r-00000' into table apis.api_logs"echo "delete /api/put/${filename}"hadoop fs -rm /api/put/${filename}echo "delete /api/out/${name}"hadoop fs -rmr /api/out/${name}echo "end"~          

大致的逻辑, 接收传入的文件(含路径), 然后分割, 得到文件名, 然后将文件put到hadoop, 调用LogMapReduce.jar, 将结果插入到hive, 删除文件

运行执行的脚本 process_2016_06.sh:

#!/bin/shfor((i=5;i<31;i++))do        logdate=`printf "%'.02d" $i`        ./run.sh /home/hadoop/data/2016-06-${logdate}.phpdone

也可以折腾在一起.

折腾到hive 就i可以查询了, 比如查询6月份失败率前20