Linux Hadoop的编译及安装

编译

• Hadoop的下载及解压
  # hadoop源代码的下载可以为不同版本，推荐hadoop-2.8.1版本，

hadoop:root:/opt/sourcecode:>lltotal 448328-rw-r--r--  1 root root  34523353 Dec 11 07:02 hadoop-2.8.1-src.tar.gzhadoop:root:/opt/sourcecode:>tar -xzvf hadoop-2.8.1-src.tar.gzhadoop:root:/opt/sourcecode:>ll hadoop-2.8.1-srctotal 220-rw-rw-r--  1 root root 15623 May 23  2017 BUILDING.txtdrwxr-xr-x  4 root root  4096 Dec 11 15:17 dev-supportdrwxr-xr-x  4 root root  4096 Dec 11 16:07 hadoop-assembliesdrwxr-xr-x  4 root root  4096 Dec 11 16:07 hadoop-build-toolsdrwxrwxr-x  3 root root  4096 Dec 11 16:26 hadoop-clientdrwxr-xr-x 11 root root  4096 Dec 11 16:12 hadoop-common-projectdrwxr-xr-x  3 root root  4096 Dec 11 16:26 hadoop-distdrwxr-xr-x  9 root root  4096 Dec 11 16:18 hadoop-hdfs-projectdrwxr-xr-x 10 root root  4096 Dec 11 16:24 hadoop-mapreduce-projectdrwxr-xr-x  4 root root  4096 Dec 11 16:07 hadoop-maven-pluginsdrwxr-xr-x  3 root root  4096 Dec 11 16:26 hadoop-miniclusterdrwxr-xr-x  4 root root  4096 Dec 11 16:07 hadoop-projectdrwxr-xr-x  3 root root  4096 Dec 11 16:07 hadoop-project-distdrwxr-xr-x 19 root root  4096 Dec 11 16:26 hadoop-toolsdrwxr-xr-x  4 root root  4096 Dec 11 16:22 hadoop-yarn-project-rw-rw-r--  1 root root 99253 May 23  2017 LICENSE.txt-rw-rw-r--  1 root root 15915 May 23  2017 NOTICE.txtdrwxrwxr-x  2 root root  4096 Jun  2  2017 patchprocess-rw-rw-r--  1 root root 20477 May 28  2017 pom.xml-rw-r--r--  1 root root  1366 May 20  2017 README.txt-rwxrwxr-x  1 root root  1841 May 23  2017 start-build-env.sh

# 查看Hadoop配置条件:

 hadoop:root:/opt/sourcecode/hadoop-2.8.1-src:>cat BUILDING.txt Build instructions for Hadoop----------------------------------------------------------------------------------Requirements:* Unix System* JDK 1.7+* Maven 3.0 or later* Findbugs 1.3.9 (if running findbugs)* ProtocolBuffer 2.5.0* CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac* Zlib devel (if compiling native code)* openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance)* Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs)* Internet connection for first build (to fetch all Maven and Hadoop dependencies)----------------------------------------------------------------------------------

• JAVA安装
  # 解压Java到/usr/java

hadoop:root:/home/hadoop:>cd /usr/java/hadoop:root:/usr/java:>tar -xzvf jdk-8u45-linux-x64.gz

# 修改用户组

 hadoop:root:/usr/java:>chown -R root:root jdk1.8.0_45hadoop:root:/usr/java:>lltotal 169216drwxr-xr-x 8 root root      4096 Apr 10  2015 jdk1.8.0_45-rw-r--r-- 1 root root 173271626 Dec 11 07:02 jdk-8u45-linux-x64.gz

# 修改环境变量 /etc/profile

hadoop:root:/usr/java:>vi /etc/profileexport JAVA_HOME=/usr/java/jdk1.8.0_45export PATH=$JAVA_HOME/bin:$PATHhadoop:root:/usr/java:>source /etc/profilehadoop:root:/usr/java:>java -versionjava version "1.8.0_45"Java(TM) SE Runtime Environment (build 1.8.0_45-b14)Java HotSpot(TM) 64-Bit Server VM (build 25.45-b02, mixed mode)

• maven安装到/etc/software,配置环境变量

hadoop:root:/opt/software:>unzip apache-maven-3.3.9-bin.ziphadoop:root:/opt/software:>vi /etc/profileexport MAVEN_HOME=/opt/software/apache-maven-3.3.9export MAVEN_OPTS="-Xms256m -Xmx512m"export PATH=$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATHhadoop:root:/opt/software/protobuf-2.5.0:>source /etc/profilehadoop:root:/opt/software:>mvn -versionApache Maven 3.3.9 (bb52d8502b132ec0a5a3f4c09453c07478323dc5; 2015-11-10T11:41:47-05:00)Maven home: /opt/software/apache-maven-3.3.9Java version: 1.8.0_45, vendor: Oracle CorporationJava home: /usr/java/jdk1.8.0_45/jreDefault locale: en_US, platform encoding: UTF-8OS name: "linux", version: "2.6.32-431.el6.x86_64", arch: "amd64", family: "unix"

• protobuf安装到/etc/software,配置环境变量

hadoop:root:/opt/software:>tar -xzvf protobuf-2.5.0.tar.gzhadoop:root:/opt/software/protobuf-2.5.0:>yum install -y gcc gcc-c++ make cmakehadoop:root:/opt/software/protobuf-2.5.0:>make && make installhadoop:root:/opt/software/protobuf-2.5.0:>vi /etc/profileexport PROTOC_HOME=/usr/local/protobufexport PATH=$PROTOC_HOME/bin:$FINDBUGS_HOME/bin:$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATHhadoop:root:/opt/software/protobuf-2.5.0:>source /etc/profilehadoop:root:/opt/software/protobuf-2.5.0:>protoc --versionlibprotoc 2.5.0

• Findbugs安装到/opt/software,配置环境变量

hadoop:root:/opt/software:>unzip findbugs-1.3.9.ziphadoop:root:/opt/software:>vi /etc/profile export FINDBUGS_HOME=/opt/software/findbugs-1.3.9export PATH=$FINDBUGS_HOME/bin:$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATHhadoop:root:/opt/software:>source /etc/profilehadoop:root:/opt/software:>findbugs -version1.3.9

• 其他依赖

yum install -y openssl openssl-devel svn ncurses-devel zlib-devel libtoolyum install -y snappy snappy-devel bzip2 bzip2-devel lzo lzo-devel lzop autoconf automake

• 编译 （PS：网速正常大概2-3小时）

 hadoop:root:/opt/software:>cd /opt/sourcecode/hadoop-2.8.1-srchadoop:root:/opt/sourcecode/hadoop-2.8.1-src:>mvn clean package -Pdist,native -DskipTests -Dtar

安装

• 创建使用Hadoop的用户,添加sudo权限

 hadoop:root:/opt/sourcecode/hadoop-2.8.1-src:>useradd hadoop hadoop:root:/opt/sourcecode/hadoop-2.8.1-src:>id hadoopuid=500(hadoop) gid=500(hadoop) groups=500(hadoop)hadoop:root:/opt/sourcecode/hadoop-2.8.1-src:>vi /etc/sudoershadoop  ALL=(root)      NOPASSWD:ALL

• 安装Java

• 解压Hadoop

 hadoop:root:/opt/sourcecode/hadoop-2.8.1-src/hadoop-dist/target:>mv /opt/sourcecode/hadoop-2.8.1-src/hadoop-dist/target/hadoop-2.8.1.tar.gz /opt/softwarehadoop:root:/opt/software:>tar -xzvf hadoop-2.8.1.tar.gz

• 配置文件
  core-site.xml

hadoop:hadoop:/opt/software/hadoop:>vi /opt/software/hadoop/etc/hadoop/core-site.xml <configuration>    <property>            <name>fs.defaultFS</name>            <value>hdfs://192.168.137.5:9000</value>    </property></configuration>

hdfs-site.xml

 <configuration>    <property>            <name>dfs.replication</name>            <value>1</value>    </property>    <property>        <name>dfs.namenode.secondary.http-address</name>            <value>hadoop:50090</value>    </property>    <property>        <name>dfs.namenode.secondary.https-address</name>            <value>hadoop:50091</value>                                                                                                      </property></configuration>

hadoop-env.sh

# The java implementation to use.export JAVA_HOME=/usr/java/jdk1.8.0_45

slave

 hadoop:hadoop:/opt/software/hadoop/etc/hadoop:>cat slaves hadoop

• 配置ssh信任关系

 hadoop:hadoop:/home/hadoop:>ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsahadoop:hadoop:/home/hadoop:>cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keyshadoop:hadoop:/home/hadoop:>chmod 0600 ~/.ssh/authorized_keyshadoop:hadoop:/home/hadoop/.ssh:>lltotal 16-rw------- 1 hadoop hadoop  395 Dec 14 04:46 authorized_keys-rw------- 1 hadoop hadoop 1675 Dec 14 04:43 id_rsa-rw-r--r-- 1 hadoop hadoop  395 Dec 14 04:43 id_rsa.pub-rw-r--r-- 1 hadoop hadoop 1182 Dec 14 04:59 known_hosts

• 格式化

hadoop:hadoop:/opt/software/hadoop:>bin/hdfs namenode format

• 启动HDFS

 hadoop:hadoop:/opt/software/hadoop:>sbin/start-dfs.sh