docker 构建spark集群

1． 准备工具
Hadoop-2.8.0.tar.gz，scala-2.12.2.tgz，spark-2.1.1-bin-hadoop2.7.tgz，zookeeper-3.4.6.tar.gz，jdk-8u73-linux-x64.tar.gz

2． 安装工具
一些工具都放到~/software目录下
docker pull ubuntu:14.04
docker run –name ubuntu_spark –h ubuntu  -v ~/software/:/software  –it ubuntu:14.04

#给容器安装软件
apt-get -y update 
apt-get install ssh
apt-get install vim
vim ~/.bashrc
#加入启动ssh脚本
/usr/sbin/sshd
#退出保存


#设置ssh使用root用户登录
vim /etc/ssh/sshd_config中PermitRootLogin no 改为yes


#生产访问秘钥：
cd ~/
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cd .ssh
cat id_rsa.pub >> authorized_keys
#开启ssh服务后验证是否可以使用，打印出当前时间
ssh localhost date


#安装jdk
cd /software
tar -zxvf jdk-8u73-linux-x64.tar.gz
mv jdk1.8.73 /usr/local/java
vim ~/.bashrc
#新增
export JAVA_HOME=/usr/local/java
export PATH=$PATH:$JAVA_HOME/bin
#验证是否成功
java --version




#安装scala
tar -zxvf scala-2.12.2.tgz
mv scala-2.12.2 /usr/local/scala
vim ~/.bashrc
#新增
export SCALA_HOME=/usr/local/scala
export PATH=$PATH:$SCALA_HOME/bin


#安装zookeeper
tar -zxvf zookeeper-3.4.6.tar.gz
mv zookeeper-3.4.6 ~/zookeeper
cd ~/zookeeper/conf/


cp zoo_sample.cfg zoo.cfg
vim zoo.cfg
#修改：
dataDir=/root/zookeeper/tmp
#在最后添加：
server.1=cloud1:2888:3888
server.2=cloud2:2888:3888
server.3=cloud3:2888:3888
#退出保存后，创建一个文件夹
mkdir ~/zookeeper/tmp
#再创建一个空文件
touch ~/zookeeper/tmp/myid
#最后向该文件写入ID
echo 1 > ~/zookeeper/tmp/myid




#安装hadoop
cd ~/software
tar -zxvf Hadoop-2.8.0.tar.gz
mv hadoop-2.8.0 ~/hadoop
vim ~/.bashrc
#新增
export HADOOP_HOME=/root/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
#退出保存


#修改hadoop相关配置
vim ~/hadoop/etc/hadoop/hadoop-env.sh
#修改JAVA_HOME
export JAVA_HOME=/usr/local/java




vim ~/hadoop/etc/hadoop/core-site.xml
#新增
<property>
    <name>fs.defaultFS</name>
    <value>hdfs://ns1</value>
</property>
<property>
    <name>hadoop.tmp.dir</name>
    <value>/root/hadoop/tmp</value>
</property>
<property>
    <name>ha.zookeeper.quorum</name> 
    <value>cloud1:2181,cloud2:2181,cloud3:2181</value>
</property>




vim ~/hadoop/etc/hadoop/hdfs-site.xml
#新增
<property>
    <name>dfs.nameservices</name>
    <value>ns1</value>
</property>
<property>
    <name>dfs.ha.namenodes.ns1</name>
    <value>nn1,nn2</value>
</property>
<property>
    <name>dfs.namenode.rpc-address.ns1.nn1</name>
    <value>cloud1:9000</value>
</property>
<property>
    <name>dfs.namenode.http-address.ns1.nn1</name>
    <value>cloud1:50070</value>
</property>
<property>
    <name>dfs.namenode.rpc-address.ns1.nn2</name>
    <value>cloud2:9000</value>
</property>
<property>
    <name>dfs.namenode.http-address.ns1.nn2</name>
    <value>cloud2:50070</value>
</property>
<property>
    <name>dfs.namenode.shared.edits.dir</name> 
    <value>qjournal://cloud1:8485;cloud2:8485;cloud3:8485/ns1</value>
</property>
<property>
    <name>dfs.journalnode.edits.dir</name>
    <value>/root/hadoop/journal</value>
</property>
<property>
    <name>dfs.ha.automatic-failover.enabled</name>
    <value>true</value>
</property>
<property>
    <name>dfs.client.failover.proxy.provider.ns1</name>
    <value>
    org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
    </value>
</property>
<property>
    <name>dfs.ha.fencing.methods</name>
    <value>
    sshfence
    shell(/bin/true)
    </value>
</property>
<property>
    <name>dfs.ha.fencing.ssh.private-key-files</name>
    <value>/root/.ssh/id_rsa</value>
</property>
<property>
    <name>dfs.ha.fencing.ssh.connect-timeout</name>
    <value>30000</value>
</property>




vim ~/hadoop/etc/hadoop/yarn-site.xml
#新增
<property>
    <name>yarn.resourcemanager.hostname</name>
    <value>cloud1</value>
</property>
<property> 
    <name>yarn.nodemanager.aux-services</name> 
    <value>mapreduce_shuffle</value> 
</property>




vim ~/hadoop/etc/hadoop/slaves
cloud1
cloud2
cloud3


#Spark安装及配置
cd ~/software
tar -xzvf spark-2.1.1-bin-hadoop2.7.tgz
mv spark-2.1.1-bin-hadoop2.7 ~/spark


vim ~/.bashrc
#新增
export SPARK_HOME=/root/spark
export PATH=$PATH:$SPARK_HOME/bin
#退出保存
cd ~/spark
cp ~/spark/conf/spark-env.sh.template ~/spark/conf/spark-env.sh
vim ~/spark/conf/spark-env.sh
#新增
export SPARK_MASTER_IP=cloud1 
export SPARK_WORKER_MEMORY=128m 
export JAVA_HOME=/usr/local/java 
export SCALA_HOME=/usr/local/scala 
export SPARK_HOME=/root/spark 
export HADOOP_CONF_DIR=/root/hadoop/etc/hadoop 
export SPARK_LIBRARY_PATH=$$SPARK_HOME/lib 
export SCALA_LIBRARY_PATH=$SPARK_LIBRARY_PATH 
export SPARK_WORKER_CORES=1 
export SPARK_WORKER_INSTANCES=1 
export SPARK_MASTER_PORT=7077


vim ~/spark/conf/slaves
#新增
cloud1
cloud2
cloud3


#在宿主机上提交ubuntu_spark容器为新的镜像，并打其标签为spark：
#提交ubuntu_spark容器，命令返回新镜像的编号
docker commit ubuntu_spark
#为新镜像打标签为spark
docker tag <mirror id> spark:1.0


docker run --name cloud1 -h cloud1 --add-host cloud1:172.17.0.3 --add-host cloud2:172.17.0.4 --add-host cloud3:172.17.0.5 -p 50070:50070 -p 8080:8080 -p 8088:8088 -it spark:1.0


docker run --name cloud2 -h cloud2 --add-host cloud1:172.17.0.3 --add-host cloud2:172.17.0.4 --add-host cloud3:172.17.0.5  -it spark:1.0


docker run --name cloud3 -h cloud3 --add-host cloud1:172.17.0.3 --add-host cloud2:172.17.0.4 --add-host cloud3:172.17.0.5  -it spark:1.0


#在cloud2执行
echo 2 > ~/zookeeper/tmp/myid
#在cloud3执行
echo 3 > ~/zookeeper/tmp/myid


#在所有的cloud1,cloud2,cloud3执行
~/zookeeper/bin/zkServer.sh start


#查看zookeeper运行状态
~/zookeeper/bin/zkServer.sh status


#cloud1,cloud2,cloud3实现免登陆，这里不写操作


#启动journalnode（在cloud1上启动所有journalnode，注意：是调用的hadoop-daemons.sh这个脚本，注意是复数s的那个脚本）
#在cloud1里执行
~/hadoop/sbin/hadoop-daemons.sh start journalnode
#格式化HDFS
~/hadoop/bin/hdfs namenode -format
#格式化ZK
~/hadoop/bin/hdfs zkfc -formatZK
#启动HDFS
~/hadoop/sbin/start-dfs.sh
~/hadoop/sbin/start-yarn.sh
#启动spark集群
~/spark/sbin/start-all.sh


启动之后可以在宿主机的浏览器中访问 
HDFS：cloud1:50070 
YARN：cloud1:8088 

SPARK：cloud1:8080

参考资料：

http://blog.csdn.net/qq1010885678/article/details/46353101

http://www.cnblogs.com/jasonfreak/p/5391190.html