Hadoop自定义数据类型

来源：互联网发布：陀思妥耶夫斯基知乎编辑：程序博客网时间：2024/05/22 00:45

Hadoop的自定制数据类型有两种，一种较为简单的是针对值，另外一种更为完整针对于键和值都适合

一、针对值，实现 Writable 接口

?
1
2
3
4
5
6
7
8
9
10
packageorg.apache.hadoop.io;
 
importjava.io.DataOutput;
importjava.io.DataInput;
importjava.io.IOException;
 
publicinterface Writable {
  voidwrite(DataOutput out) throwsIOException;
  voidreadFields(DataInput in) throwsIOException;
}

例子：

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
importorg.apache.hadoop.io.Writable;
importjava.io.DataInput;
importjava.io.DataOutput;
importjava.io.IOException;
 
publicclass Address implementsWritable {
    publicString city;
    publicString street;
    publicint doorplate;
 
    publicAddress() {
        this("","",0);
    }
 
    publicAddress(String city, String street, intdoorplate) {
        this.city = city;
        this.street = street;
        this.doorplate = doorplate;
    }
 
    @Override
    publicvoid write(DataOutput out) throwsIOException {
        out.writeUTF(this.city);
        out.writeUTF(this.street);
        out.writeInt(this.doorplate);
    }
 
    @Override
    publicvoid readFields(DataInput in) throwsIOException {
        this.city = in.readUTF();
        this.street = in.readUTF();
        this.doorplate = in.readInt();
    }
 
    publicString toString() {
        returnthis.city + ","+ this.street + ","+ this.doorplate;
    }
}

二、针对于键和值，需要指定排序规则，自定义类需要实现 WritableComparable 泛型接口

?
1
2
3
4
5
6
7
8
9
packageorg.apache.hadoop.io;
publicinterface WritableComparable<T> extendsWritable, Comparable<T> {
}
 
packagejava.lang;
importjava.util.*;
publicinterface Comparable<T> {
    publicint compareTo(T o);
}

例子：

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
importorg.apache.hadoop.io.WritableComparable;
importjava.io.DataInput;
importjava.io.DataOutput;
importjava.io.IOException;
 
publicclass Address implementsWritableComparable {
    publicString city;
    publicString street;
    publicint doorplate;
 
    publicAddress() {
        this("","",0);
    }
 
    publicAddress(String city, String street, intdoorplate) {
        this.city = city;
        this.street = street;
        this.doorplate = doorplate;
    }
 
    @Override
    publicvoid write(DataOutput out) throwsIOException {
        out.writeUTF(this.city);
        out.writeUTF(this.street);
        out.writeInt(this.doorplate);
    }
 
    @Override
    publicvoid readFields(DataInput in) throwsIOException {
        this.city = in.readUTF();
        this.street = in.readUTF();
        this.doorplate = in.readInt();
    }
 
    @Override
    publicint compareTo(Object o) {
        Address other = (Address) o;
        intn = this.strCompareTo(this.city, other.city);
        if(n != 0)returnn;
        n = this.strCompareTo(this.street, other.street);
        if(n != 0)returnn;
        return(this.doorplate < other.doorplate ? -1: (this.doorplate == other.doorplate ? 0: 1));
    }
 
    publicString toString() {
        returnthis.city + ","+ this.street + ","+ this.doorplate;
    }
 
    publicboolean equals(Object obj) {
        if(this== obj) {
            returntrue;
        }
        if(obj instanceofAddress) {
            Address other = (Address) obj;
            returnthis.strEquals(this.city, other.city)
                    &&this.strEquals(this.street, other.street)
                    &&this.doorplate == other.doorplate;
        }
        returnfalse;
    }
 
    /**
     * 重写 hashCode() 方法很重要，Hadoop 的 Partitioners 会用到这个方法
     */
    publicint hashCode() {
        return13 * (this.city == null? 0: this.city.hashCode())
                +67* (this.street == null? 0: this.street.hashCode())
                +151* this.doorplate;
    }
 
    publicboolean strEquals(String a, String b) {
        return(a == null&& b == null) || (a != null&& a.equals(b));
    }
 
    publicint strCompareTo(String a, String b) {
        if(a == null&& b == null)return0;
        elseif (a != null&& b == null)return1;
        elseif (a == null&& b != null)return-1;
        elsereturn a.compareTo(b);
    }
}

0 0