在Windows 调试 Hadoop程序

转：http://blog.csdn.net/uq_jin/article/details/52235121

1、解压Hadoop到任意目录

比如：D:\soft\dev\Hadoop-2.7.2

2、设置环境变量

HADOOP_HOME：D:\soft\dev\hadoop-2.7.2HADOOP_BIN_PATH：%HADOOP_HOME%\binHADOOP_PREFIX：%HADOOP_HOME%在Path后面加上%HADOOP_HOME%\bin;%HADOOP_HOME%\sbin;

把 hadoop-2.7.3\share\hadoop\hdfs\hadoop-hdfs-2.7.3.jar，share\hadoop\hdfs\lib

D:\BaiduYunDownload\hadoop-2.7.3\share\hadoop\common\hadoop-common-2.7.3.jar，\share\hadoop\common\lib

的jar包拷贝到项目lib下

HdfsClientTest

package com.hadoop.test;import java.io.FileInputStream;import java.io.FileOutputStream;import java.io.IOException;import java.net.URI;import org.apache.commons.io.IOUtils;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.junit.Before;import org.junit.Test;public class HdfsClientTest {private FileSystem fs = null;@Beforepublic void setUp() throws Exception {// get a configuration objectConfiguration conf = new Configuration();// to set a parameter,figure out the filesystemis hdfsconf.set("fs.defalutFS", "hdfs://192.168.169.128:9000");conf.set("dfs.replication", "1");// get a instance of HDFS FileSystem Clientfs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");}@Testpublic void testDownload() throws IllegalArgumentException, IOException {FSDataInputStream is = fs.open(new Path("hdfs://192.168.169.128:9000/hadoop-common-2.7.3.jar"));FileOutputStream os = new FileOutputStream("D:/hdfs/down.apk");IOUtils.copy(is, os);}// 上传文件public static void main(String[] args) throws Exception {// get a configuration objectConfiguration conf = new Configuration();// to set a parameter,figure out the filesystemis hdfsconf.set("fs.defalutFS", "hdfs://192.168.169.128:9000");conf.set("dfs.replication", "1");// get a instance of HDFS FileSystem ClientFileSystem fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");// open a outputstream of the dest filePath destFile = new Path("hdfs://192.168.169.128:9000/hadoop-common-2.7.3.jar");FSDataOutputStream os = fs.create(destFile);// open a inputstream of the local source fileFileInputStream is = new FileInputStream("D:/BaiduYunDownload/hadoop-2.7.3/share/hadoop/common/hadoop-common-2.7.3.jar");// write the bytes in "is" to "os"IOUtils.copy(is, os);}}

HdfsClientEasy

package com.hadoop.test;import static org.junit.Assert.*;import java.io.FileNotFoundException;import java.io.IOException;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.LocatedFileStatus;import org.apache.hadoop.fs.Path;import org.apache.hadoop.fs.RemoteIterator;import org.junit.Before;import org.junit.Test;public class HdfsClientEasy {private FileSystem fs = null;@Beforepublic void setUp() throws Exception {// 拿到一个配置参数的封装对象，构造函数中就会对classpath下的xxx-site.xml文件进行解析// 真实的项目工程中就应该把xxx-site.xml文件加入到工程中来Configuration conf = new Configuration();// to set a parameter,figure out the filesystemis hdfsconf.set("fs.defalutFS", "hdfs://192.168.169.128:9000");conf.set("dfs.replication", "1");// 这种获取fs的方法可以指定访问hdfs的客户端身份fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");}/** * 上传文件 *  * @throws IllegalArgumentException * @throws IOException */@Testpublic void testUpload() throws IllegalArgumentException, IOException {fs.copyFromLocalFile(new Path("D:/HooShell_Shudu.apk"),new Path("/hadoop-common-2.7.3.jar"));}/** * 删除文件 *  * @throws IllegalArgumentException * @throws IOException */@Testpublic void testRmfile() throws IllegalArgumentException, IOException {boolean res = fs.delete(new Path("/hadoop-common-2.7.3.jar"), true);System.out.println(res ? "delete is successfully :)" : "it is failed :(");}/** * 创建文件夹 *  * @throws IllegalArgumentException * @throws IOException */@Testpublic void testMkdir() throws IllegalArgumentException, IOException {fs.mkdirs(new Path("/aa/bb"));}/** * 重命名文件 *  * @throws IllegalArgumentException * @throws IOException */@Testpublic void testRename() throws IllegalArgumentException, IOException {fs.rename(new Path("/hadoop-common-2.7.3.jar"), new Path("/hadoop-common.jar"));}@Testpublic void testListFiles() throws FileNotFoundException, IllegalArgumentException, IOException {RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);while (listFiles.hasNext()) {LocatedFileStatus file = listFiles.next();System.out.println(file.getPath().getName());}System.out.println("--------------------------------------------");// 列出文件及文件夹FileStatus[] status = fs.listStatus(new Path("/"));for (FileStatus file : status) {System.out.println(file.getPath().getName() + "   " + (file.isDirectory() ? "d" : "f"));}}/** * 从hdfs中下载数据到本地 *  * @throws IllegalArgumentException * @throws IOException */@Testpublic void testDownload() throws IllegalArgumentException, IOException {// fs.c// fs.copyToLocalFile(true,new Path("/hadoop-common-2.7.3.jar"), new// Path("d:/jdk12333.win"));fs.copyToLocalFile(false, new Path("/hadoop-common-2.7.3.jar"), new Path("c:/jdk.win"), true);}}

出现的问题

Permission denied: user=root, access=WRITE, inode="/user/hive/warehouse":hadoop:hadoop:drwxrwxr-x

原因：本地用户administrator（本机windows用户）想要远程操作hadoop系统，没有权限引起的。

 解决办法：

1、如果是测试环境，可以取消hadoop hdfs的用户权限检查。打开conf/hdfs-site.xml，找到dfs.permissions属性修改为false（默认为true）OK了。

2. 指定访问hdfs的客户端身份
fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");



fs.copyFromLocalFile(new Path("D:/HooShell_Shudu.apk"),new Path("/hadoop-common-2.7.3.jar"));报错
org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /HooShell_Shudu.apk could only be replicated to 0 nodes instead of minReplication (=1).  There are 1 datanode(s) running and 1 node(s) are excluded in this operation.
    at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:1571)
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getNewBlockTargets(FSNamesystem.java:3107)
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3031)
    at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:725)

解决方法:
#关闭防火墙
sudo systemctl stop firewalld.service
#关闭开机启动
sudo systemctl disable firewalld.service    
    
    
java.io.IOException: (null) entry in command string: null chmod 0644 D:\hdfs\jdk123.win
    at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:769)
    at org.apache.hadoop.util.Shell.execCommand(Shell.java:866)
    at org.apache.hadoop.util.Shell.execCommand(Shell.java:849)
    at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:733)
    at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:225)
    at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:209)
    at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:307)

解决方法:将fs.copyToLocalFile( hdfsPath,localPath);改为fs.copyToLocalFile( false,hdfsPath,localPath,true);
原因:不理解,但加上这两个后确实可以下载