hadoop由于NodeManager无法启动而导致执行Jar包出现running job卡住的解决方案之一...

来源：互联网发布：数据库考点总结编辑：程序博客网时间：2024/06/05 02:58

菜鸟上手大数据一头雾水...所以先自己用三个虚拟机搭建一个hadoop集群环境了

hadoop版本2.7.3

JDK版本8u101

系统CentOS7

基本按照博文

其中有两个值得注意的地方(或者说勘误一下)：

其一第8步的mapred-site.xml应为yarn-site.xml

其二还是第8步中关于yarn.nodemanager.resource.memory-mb的设置，请设置至少为1024，否则host的nodemanager会启动失败

hadoop集群正常启动的情况下，可以用jps查看服务的端口，其中的master应该至少启动了

SecondaryNameNode
NameNode
ResourceManager
Jps

host在没有执行任务时应该至少启动了

Jps
DataNode
NodeManager

设置完毕后执行wordcount的程序，发现还是卡住了，调hadoop的日志看(日志的位置在start-all.sh)的时候会显示，默认在hadoop文件夹的log中看yarn的日志)发现分配的空间小了，我的wordcount需要1500mb空间，而设置成1024不够，于是再次将 yarn.nodemanager.resource.memory-mb都设置为2048后程序跑通，出来了想要的wordcount结果。

其中多次重新设置hadoop,我都选择了清空之前的文件以及格式化namenode,否则容易报错。可能有些暴力...等熟悉后有更好的重新设置并启动的方案吧。

最后贴个我的wordcount留作纪念....向大数据说helloword!

/** * Created by jacobzhou on 2016/8/29. */import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;import java.io.IOException;public class HadoopTest extends Configured implements Tool {    private static String DELIMA = "\t";    public static class MRMapper extends Mapper<Object, Text, Text, LongWritable> {        protected void map(Object key, Text value, Mapper.Context context) throws IOException, InterruptedException {                context.write(value,new LongWritable(1));        }        protected void cleanup(Context context) throws IOException, InterruptedException {}    }    public static class MRReducer extends Reducer<Text,LongWritable,Text,LongWritable> {        private LongWritable result = new LongWritable();        public void reduce(Text key, Iterable<LongWritable> values, Context context)                throws IOException, InterruptedException {                long sum = 0;                for (LongWritable val : values) {                    sum += val.get();                }                result.set(sum);                context.write(key, result);        }    }    public int run(String[] args) throws Exception {        Configuration conf = getConf();        conf.set("mapreduce.job.queuename", "datacenter");        conf.set("mapred.max.map.failures.percent", "5");        int reduceTasksMax = 10;        Job job = new Job(conf);        job.setJobName("wordCount job");        job.setNumReduceTasks(reduceTasksMax);        job.setMapOutputKeyClass(Text.class);        job.setMapOutputValueClass(LongWritable.class);        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(LongWritable.class);        job.setMapperClass(MRMapper.class);        job.setReducerClass(MRReducer.class);        job.setJarByClass(HadoopTest.class);        job.setOutputFormatClass(TextOutputFormat.class);        FileInputFormat.addInputPath(job,new Path(args[0]));        TextOutputFormat.setOutputPath(job, new Path(args[1]));        return job.waitForCompletion(true) ? 0 : 1;    }    public static void main(String[] args) throws Exception {        try{            System.out.println("start run job!");            int ret = ToolRunner.run(new HadoopTest(), args);            System.exit(ret);        }catch (Exception e){            e.printStackTrace();        }    }}

0 0