hadoop自定义数据类型

测试类

package com.Datatype;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class DataLogs1 {

public static class Mappeer extends Mapper<LongWritable, Text, Log1, Text> {

@Override

protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

System.out.println("******************************");

String logs = value.toString();

String[] datas = logs.split(" ");

System.out.println(datas[datas.length - 2]);

if (datas[datas.length - 2].equals("404")) {

Log1 log = new Log1(logs, datas[3], datas[datas.length - 1]);

System.out.println(log);

context.write(log, new Text("1"));

}

}

}

public static class Reduceer extends Reducer<Log1, Text, Log1, IntWritable> {

@Override

protected void reduce(Log1 log, Iterable<Text> value, Context arg2) throws IOException, InterruptedException {

int sum = 0;

System.out.println(sum);

for (Text text : value) {

sum = sum + Integer.parseInt(text.toString());

}

arg2.write(log, new IntWritable(sum));

}

}

public static void main(String[] args) {

Configuration conf = new Configuration();

conf.set("fs.defaultFS", "hdfs://192.168.10.97:9000/");

Job job;

try {

job = new Job(conf);

job.setJarByClass(DataLogs1.class);

job.setMapperClass(Mappeer.class);

job.setReducerClass(Reduceer.class);

job.setMapOutputKeyClass(Log1.class);

job.setMapOutputValueClass(Text.class);

job.setOutputKeyClass(Log1.class);

job.setOutputValueClass(IntWritable.class);

FileInputFormat.addInputPath(job, new Path("/root/log.txt"));

FileOutputFormat.setOutputPath(job, new Path("/file/test4111"));

System.out.println("3333333333");

try {

System.exit(job.waitForCompletion(true) ? 0 : 1);

// job.waitForCompletion(true);

} catch (ClassNotFoundException e) {

System.out.println("任务提交出了问题 ClassNotFoundException");

e.printStackTrace();

} catch (InterruptedException e) {

System.out.println("任务提交出了问题 InterruptedException");

e.printStackTrace();

}

} catch (IOException e) {

System.out.println("job初始化出了问题");

e.printStackTrace();

}

}

}

自定义数据类型

无参构造方法必须要有：初始化要用

package com.Datatype;

import java.io.DataInput;

import java.io.DataOutput;

import java.io.IOException;

import org.apache.hadoop.io.WritableComparable;

public class Log1 implements WritableComparable<Log1> {

private String data;

private String time;

private String flag;

public Log1(){}

public Log1(String data, String time, String flag) {

this.data = data;

this.time = time;

this.flag = flag;

}

public String getData() {

return data;

}

public void setData(String data) {

this.data = data;

}

public String getTime() {

return time;

}

public void setTime(String time) {

this.time = time;

}

public String getFlag() {

return flag;

}

public void setFlag(String flag) {

this.flag = flag;

}

@Override

public String toString() {

return data;

}

@Override

public void readFields(DataInput in) throws IOException {

this.data = in.readUTF();

this.time = in.readUTF();

this.flag = in.readUTF();

}

@Override

public void write(DataOutput out) throws IOException {

out.writeUTF(data);

out.writeUTF(time);

out.writeUTF(flag);

}

@Override

public int compareTo(Log1 log) {

if (this.flag.compareTo(log.flag) == 0) {

if (this.time.compareTo(log.time) == 0) {

return 0;

} else {

return this.time.compareTo(log.time) > 0 ? -1 : 1;

}

} else {

return this.flag.compareTo(log.flag) > 0 ? -1 : 1;

}

}

}