Hadoop环境搭建

1. 准备工作
   虚拟机（3个）、镜像文件、hadoop包、jdk包
   服务器功能规划
   hadoop001:NameNode,DataNode,NodeManager,HistoryServer
   hadoop002:ResourceManager,DataNode,NodeManager
   hadoop003:DataNode,NodeManager,SecondaryNameNode
2. 搭建步骤
   1、创建hadoop用户并配置sudo权限
   [hadoop]# su
   Password:hadoop
   [hadoop]# chmod u+w /etc/sudoers （给sudoers增加写权限）
   [hadoop]# vi /etc/sudoers （修改sudoers文件）
   [hadoop]# hadoop ALL=(ALL)NOPASSWD:ALL
   [hadoop]# chmod u-w /etc/sudoers （撤销sudoers文件的写权限）
   [hadoop]# ip address（查看ip地址）
   [hadoop]# sudo chmod 777 /etc/sysconfig/network-scripts/ifcfg-eth0
   [hadoop]# vi /etc/sysconfig/network-scripts/ifcfg-eth0
   
   [hadoop]# sudo service network restart
   [hadoop]# sudo vi /etc/hosts(修改主机名)
   192.168.150.200 hadoop001
   [hadoop]# sudo vi /etc/sysconfig/network
   HOSTNAME=hadoop001
   2、安装JDK
   [hadoop]# sudo vi /etc/profile
   export JAVA_HOME=/home/hadoop/jdk1.8.0_131
   export CLASSPATH=.:JAVAHOME/jre/lib/rt.jar:JAVA_HOME/lib/dt.jar:JAVAHOME/lib/tools.jarexportPATH=PATH:$JAVA_HOME/bin
   [hadoop]# source /etc/profile
   3、安装hadoop
   [hadoop]# cd /home/hadoop/hadoop-2.8.1/etc/hadoop
   在hadoop-env.sh、mapred-env.sh、yarn-env.sh配置如下：
   export JAVA_HOME=/home/hadoop/jdk1.8.0_131
   配置core-site.xml：
   
   fs.defaultFS
   hdfs://hadoop001:8020
   
   
   hadoop.tmp.dir
   /home/hadoop/hadoop-2.8.1/data/tmp
   
   配置hdfs-site.xml：
   
   dfs.namenode.secondary.http-address
   hadoop003:50090
   
   配置slaves：
   [hadoop]# vi slaves
   配置yarn-site.xml：
   
   yarn.nodemanager.aux-services
   mapreduce_shuffle
   
   
   yarn.resourcemanager.hostname
   hadoop002
   
   
   yarn.log-aggregation-enable
   true
   
   
   yarn.log-aggregation.retain-seconds
   106800
   
   配置mapred-site.xml：
   
   mapreduce.framework.name
   yarn
   
   
   mapreduce.jobhistory.address
   hadoop001:10020
   
   
   mapreduce.jobhistory.webapp.address
   hadoop001:19888
   
   设置SSH免密登录
   [hadoop]# ssh-keygen -t rsa
   [hadoop]# ssh-copy-id hadoop001
   [hadoop]# ssh-copy-id hadoop002
   [hadoop]# ssh-copy-id hadoop003
   分发hadoop文件(先删除doc文件)
   [hadoop]# scp -r hadoop hadoop002:/home/hadoop
   格式NameNode
   在hadoop001上：
   [hadoop]# hdfs namenode -format
   启动集群：
   hadoop001：start-dfs.sh,start-yarn.sh,./mr-jobhistory-daemon.sh start historyserver
   hadoop002：./yarn-daemon.sh start resourcemanager
   注意：查看防火墙状态：firewall-cmd –state
   systemctl stop firewalld.service #停止firewall
   systemctl disable firewalld.service #禁止firewall开机启动
   查看HDFSWEB页面
   hadoop001:50070
   查看YARNWEB页面
   hadoop002:8088/cluster