java.io.IOException: Illegal partition for 26 (-1)

来源：互联网发布：1元洗车软件编辑：程序博客网时间：2024/04/30 14:45

14/08/01 15:21:03 INFO mapred.JobClient: Task Id : attempt_201408011020_0012_m_000002_0, Status : FAILED
java.io.IOException: Illegal partition for 26 (-1)
at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.collect(MapTask.java:1078)
at org.apache.hadoop.mapred.MapTask$NewOutputCollector.write(MapTask.java:690)
at org.apache.hadoop.mapreduce.TaskInputOutputContext.write(TaskInputOutputContext.java:80)
at com.hadoop.examples.Sort$Map.map(Sort.java:32)
at com.hadoop.examples.Sort$Map.map(Sort.java:1)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:764)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:364)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190)
at org.apache.hadoop.mapred.Child.main(Child.java:249)

14/08/01 15:21:04 INFO mapred.JobClient: Task Id : attempt_201408011020_0012_m_000000_0, Status : FAILED
java.io.IOException: Illegal partition for 2 (-1)
at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.collect(MapTask.java:1078)
at org.apache.hadoop.mapred.MapTask$NewOutputCollector.write(MapTask.java:690)
at org.apache.hadoop.mapreduce.TaskInputOutputContext.write(TaskInputOutputContext.java:80)
at com.hadoop.examples.Sort$Map.map(Sort.java:32)
at com.hadoop.examples.Sort$Map.map(Sort.java:1)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:764)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:364)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190)
at org.apache.hadoop.mapred.Child.main(Child.java:249)

14/08/01 15:21:04 INFO mapred.JobClient: Task Id : attempt_201408011020_0012_m_000001_0, Status : FAILED
java.io.IOException: Illegal partition for 5956 (-1)
at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.collect(MapTask.java:1078)
at org.apache.hadoop.mapred.MapTask$NewOutputCollector.write(MapTask.java:690)
at org.apache.hadoop.mapreduce.TaskInputOutputContext.write(TaskInputOutputContext.java:80)
at com.hadoop.examples.Sort$Map.map(Sort.java:32)
at com.hadoop.examples.Sort$Map.map(Sort.java:1)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:764)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:364)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190)
at org.apache.hadoop.mapred.Child.main(Child.java:249)

/**

* 自定义Partition函数，此函数根据输入数据的最大值和MapReduce框架中
* Partition的数量获取将输入数据安装大小分块的边界，然后根据输入数据和边界的关系
* 返回对象Partition ID
*/
public static class Partition extends Partitioner<IntWritable, IntWritable>
{
@Override
public int getPartition(IntWritable key, IntWritable value, int numPartitions)
{
int maxNumber = 65223;
int bound = maxNumber / numPartitions + 1;
int keyNumber = key.get();
for(int i = 0; i < numPartitions; i++)
{
if(keyNumber < bound * i && keyNumber >= bound * (i - 1))
{
return i - 1;
}
}
return 0;
//return -1;
}
}

0 0