windows Eclipse远程连接linux hadoop2.3.0伪分布式
来源:互联网 发布:最好的网络机顶盒 编辑:程序博客网 时间:2024/06/17 12:22
系统:win7 32位 Eclipse-luna
hadoop:centos hadoop2.3.0伪分布式
程序为hadoop2.3.0中的wordcount源码
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
public class WordCount {
public static class TokenizerMapper
extends Mapper<Object, Text, Text, IntWritable>{
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken());
context.write(word, one);
}
}
}
public static class IntSumReducer
extends Reducer<Text,IntWritable,Text,IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values,
Context context
) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: wordcount <in> <out>");
System.exit(2);
}
Job job = new Job(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
// FileInputFormat.addInputPath(job, new Path("/test_in"));
// FileOutputFormat.setOutputPath(job, new Path("/out1"));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
hdfs上传文件正常,但是运行该程序出现错误:
log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread "main" java.lang.NullPointerException
at java.lang.ProcessBuilder.start(ProcessBuilder.java:1012)
at org.apache.hadoop.util.Shell.runCommand(Shell.java:445)
at org.apache.hadoop.util.Shell.run(Shell.java:418)
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:650)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:739)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:722)
at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:631)
at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:421)
at org.apache.hadoop.fs.FilterFileSystem.mkdirs(FilterFileSystem.java:277)
at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:125)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:348)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1285)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1282)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1282)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1303)
at WordCount.main(WordCount.java:66)
66行为程序的最后一句。
我的win7是32位,在网上下载32位的winutils.exe和hadoop.dll放到hadoop的bin文件夹下,并将hadoop.dll放到C:/Windows/System32下,并配置了win7下的HADOOP_HOME和指向bin的PATH。仍然报错:
log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Exception in thread "main" org.apache.hadoop.util.Shell$ExitCodeException:
at org.apache.hadoop.util.Shell.runCommand(Shell.java:505)
at org.apache.hadoop.util.Shell.run(Shell.java:418)
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:650)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:739)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:722)
at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:631)
at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:421)
at org.apache.hadoop.fs.FilterFileSystem.mkdirs(FilterFileSystem.java:277)
at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:125)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:348)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1285)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1282)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1548)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1282)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1303)
at WordCount.main(WordCount.java:66)
在cmd里输入winutils.exe回车,提示系统缺少MSVCR100D.dll,在网上下载该dll,放在C:\Windows\System32中。
问题解决!!!(若还不能解决,再重启电脑试试看)
折腾了好久,看到不报错时候,简直泪奔
- windows Eclipse远程连接linux hadoop2.3.0伪分布式
- windows10+eclipse neon+hadoop2.6.4(伪分布式)远程连接虚拟机环境搭建
- windows eclipse远程连接hadoop2.6
- windows eclipse 远程连接 hadoop(linux)
- windows eclipse 远程连接 hadoop(linux)
- Hadoop 远程连接伪分布式Hadoop异常
- Hadoop2 伪分布式部署
- hadoop2.4.1伪分布式
- Hadoop2伪分布式安装
- Hadoop2 伪分布式安装
- Windows上Eclipse远程调用Hadoop2.6.0
- windows+vmware+centos7+hadoop2.7搭建伪分布式集群
- Hadoop I 搭建Linux下Hadoop2.6.0伪分布式环境
- linux RedHat系统版本伪分布式安装hadoop2.5.2
- Linux环境Hadoop2.8.0搭建(单机模式-伪分布式模式)
- 基于linux centos6.5 hadoop2.x伪分布式搭建
- 在Linux环境下部署Hadoop2.4.1伪分布式
- Linux安装Hadoop2.4.1和伪分布式部署
- 计算两点间的距离_Java_杭电ACM2001
- AFG与AWG的比较
- 内存管理
- TestNG+Ant 测试报告美化
- iOS进行单元测试OCUnit+xctool
- windows Eclipse远程连接linux hadoop2.3.0伪分布式
- 一次mysql数据库迁移的过程记录
- LNUX 介绍各个文件目录
- 如何用Python写一个贪吃蛇AI
- Android开发(28) Fragment生命周期
- javascript基础
- Use gitk to understand git – merge and rebase
- 解决压缩包内乱码问题
- Dreamweaver 8 锚点链接 下载链接和空链接