使用Java程序来获得某个集群的NameNode或DataNode的相关信息

来源：互联网发布：centos 安装中文包编辑：程序博客网时间：2024/05/16 03:21

http://www.aboutyun.com/thread-18336-1-1.html

软件版本：hadoop2.2，JDK1.7，Intellij idea14

0. 配置Configuration

如果需要使用Java程序来获得某个集群的NameNode或DataNode的相关信息，那么Configuration需要如下配置：

[Java] 纯文本查看复制代码

?
01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
16
17
/**
        * 获得配置的Configuration 
        * 
        * @return 
        */ 
       publicstatic Configuration getConf() {  
           if(conf == null) {  
               conf = newConfiguration();  
               conf.setBoolean("mapreduce.app-submission.cross-platform",true);// 配置使用跨平台提交任务  
               conf.set("fs.defaultFS","hdfs://"+ namenode + ":8020");// 指定namenode  
               conf.set("mapreduce.framework.name","yarn");// 指定使用yarn框架  
               conf.set("yarn.resourcemanager.address", resourcenode + ":8032");// 指定resourcemanager  
               conf.set("yarn.resourcemanager.scheduler.address", schedulernode + ":8030");// 指定资源分配器  
               conf.set("mapreduce.jobhistory.address", jobhistorynode + ":10020");// 指定historyserver  
           } 
           returnconf;  
       }

1. 针对Hadoop集群（不管有没有配置HA），获得DataNode相关信息；

[Java] 纯文本查看复制代码

?
01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
16
17
18
/**
     * 获取子节点
     * @return
     */
    publicstatic DatanodeInfo getDATANODE() {
        if(DATANODE==null) {
            try{
                DatanodeInfo[] live =  ((DistributedFileSystem) getFS()).getDataNodeStats(HdfsConstants.DatanodeReportType.LIVE);
                if(live.length<1){
                    log.error("获取live子节点失败!");
                }
                DATANODE=live[0];
            }catch(IOException e){
                log.warn("获取live子节点异常！\n"+e.getMessage());
            }
        }
        returnDATANODE;
    }

2. 针对配置了HA的Hadoop集群，获得Active NameNode的代码：

[Java] 纯文本查看复制代码

?
01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
packageorg.apache.hadoop.hdfs.tools;
importorg.apache.hadoop.conf.Configuration;
importorg.apache.hadoop.fs.CommonConfigurationKeys;
importorg.apache.hadoop.ha.HAServiceProtocol;
importorg.apache.hadoop.ha.HAServiceStatus;
importorg.apache.hadoop.ha.HAServiceTarget;
importorg.slf4j.Logger;
importorg.slf4j.LoggerFactory;
importjava.io.IOException;
/**
* 动态获得Active NameNode
* 针对HA集群
* Created by fansy on 2016/3/29.
*/
publicclass DFSHAAdminUtils extendsDFSHAAdmin {
    privateLogger log = LoggerFactory.getLogger(DFSHAAdminUtils.class);
    privateint rpcTimeoutForChecks = -1;
    private String nnidStr;
    privatestatic String activeNameNodeHostAndPort=null;
    publicDFSHAAdminUtils(){}
    publicDFSHAAdminUtils(Configuration conf,String nnidStr){
        super.setConf(conf);
        if(conf != null) {
            rpcTimeoutForChecks = conf.getInt(
                    CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY,
                    CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT);
        }
        this.nnidStr= nnidStr;
    }
    publicString getActiveNameNode(){
        String[] nnids = nnidStr.split(",",-1);
        HAServiceProtocol proto = null;
        HAServiceTarget haServiceTarget=null;
        HAServiceStatus state= null;
        try{
            for(String nnid : nnids){
                haServiceTarget = resolveTarget(nnid);
                proto = haServiceTarget.getProxy(
                        getConf(), rpcTimeoutForChecks);
                state = proto.getServiceStatus();
                if(state.getState().ordinal()==HAServiceProtocol.HAServiceState.ACTIVE.ordinal()) {
                    log.info("Active NameNode:{}",
                            haServiceTarget.getAddress().getHostName() + ":"+ haServiceTarget.getAddress().getPort());
                    returnhaServiceTarget.getAddress().getHostName()+":"+haServiceTarget.getAddress().getPort();
                }
            }
        }catch(IOException e){
            log.error("获取Active NameNode异常!");
        }
        returnnull;
    }
}

其中，DFSHAAdmin是Hadoop自带的，同时，这里的实现也是参考了其内部的实现；

0 0