rhel6多台主机的HA集群，并实现增加仲裁盘和共享存储

所有的服务最好设置开机启动
[root@node1-f15 ~]# chkconfig modclusterd on
[root@node1-f15 ~]# chkconfig ricci on
[root@node1-f15 ~]# chkconfig luci on
[root@node1-f15 ~]# chkconfig cman on
[root@node1-f15 ~]# chkconfig rgmanager on


[root@node2-f15 ~]#  vim /etc/cluster/cluster.conf 
[root@node2-f15 ~]# chkconfig modclusterd on
[root@node2-f15 ~]# chkconfig ricci on
[root@node2-f15 ~]# chkconfig cman on
[root@node2-f15 ~]#  chkconfig rgmanager on


[root@node3-f15 ~]# chkconfig modclusterd on
[root@node3-f15 ~]# chkconfig ricci on
[root@node3-f15 ~]# chkconfig cman on
[root@node3-f15 ~]#  chkconfig rgmanager on


[root@node4-f15 ~]# chkconfig tgtd on
[root@node4-f15 ~]# chkconfig nfs on




--------------------------------------------------------------------------------------------
[root@node1-f15 ~]# vim /etc/cluster/cluster.conf #修改配置文件，version增加1，增加主机node3
[root@node1-f15 ~]# cman_tool status #查看
[root@node1-f15 ~]# cman_tool -r version #将配置文件推送到其他主机
[root@node1-f15 ~]# cman_tool status


[kiosk@foundation15 Desktop]$ ssh root@172.25.15.12#下载初始化环境脚本，也可以自己配置新环境
Last login: Thu Jul  2 16:02:25 2015 from 172.25.0.250
[root@rhel6 ~]# wget http://classroom.example.com/content/ula/cluster/clusterinit.sh
--2016-09-28 14:02:54--  http://classroom.example.com/content/ula/cluster/clusterinit.sh
Resolving classroom.example.com... 172.25.254.254
Connecting to classroom.example.com|172.25.254.254|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 2550 (2.5K) [application/x-sh]
Saving to: “clusterinit.sh”


100%[======================================>] 2,550       --.-K/s   in 0s      


2016-09-28 14:02:54 (5.76 MB/s) - “clusterinit.sh” saved [2550/2550]


[root@rhel6 ~]# bash clusterinit.sh node3-f15.example.com#初始化环境
Stopping NetworkManager daemon:                            [  OK  ]
iptables: Saving firewall rules to /etc/sysconfig/iptables:[  OK  ]
[root@rhel6 ~]# reboot #初始化完毕后重启主机


Broadcast message from [e-mail]
(/dev/pts/0) at 14:03 ...


The system is going down for reboot NOW!
[kiosk@foundation15 Desktop]$ ssh root@172.25.15.12
Last login: Wed Sep 28 14:02:28 2016 from 172.25.15.250
[root@node3-f15 ~]# yum install ricci -y #安装客户端软件
                                               


Dependency Installed:
  clusterlib.x86_64 0:3.0.12.1-59.el6     corosync.x86_64 0:1.4.1-17.el6       
  corosynclib.x86_64 0:1.4.1-17.el6       libibverbs.x86_64 0:1.1.7-1.el6      
  librdmacm.x86_64 0:1.0.17-1.el6         modcluster.x86_64 0:0.16.2-28.el6    


Complete!
[root@node3-f15 ~]#  vim /etc/sysconfig/network-scripts/ifcfg-eth2
[root@node3-f15 ~]#  vim /etc/sysconfig/network-scripts/ifcfg-eth0
[root@node3-f15 ~]#  vim /etc/sysconfig/network-scripts/ifcfg-eth1




[root@node3-f15 ~]#  passwd ricci
Changing password for user ricci.
New password: 
BAD PASSWORD: it is based on a dictionary word
Retype new password: 
passwd: all authentication tokens updated successfully.
[root@node3-f15 ~]# service ricci start #启动服务
--------------------------------------------------------------------
[root@node1-f15 ~]# cman_tool status #查看
[root@node1-f15 ~]# cman_tool -r version #将配置文件推送到其他主机
[root@node1-f15 ~]# cman_tool status
--------------------------------------------------------------------
Starting oddjobd:                                          [  OK  ]
generating SSL certificates...  done
Generating NSS database...  done
Starting ricci:                                            [  OK  ]


[root@node3-f15 ~]# yum grouplist #查看软件组包
[root@node3-f15 ~]# yum groupinstall "High Availability" -y #安装组包


[root@node3-f15 ~]# service cman start #安装后启动服务，服务为cman
Starting cluster: 
   Checking if cluster has been disabled at boot...        [  OK  ]
   Checking Network Manager...                             [  OK  ]
   Global setup...                                         [  OK  ]
   Loading kernel modules...                               [  OK  ]
   Mounting configfs...                                    [  OK  ]
   Starting cman...                                        [  OK  ]
   Waiting for quorum...                                   [  OK  ]
   Starting fenced...                                      [  OK  ]
   Starting dlm_controld...                                [  OK  ]
   Tuning DLM kernel config...                             [  OK  ]
   Starting gfs_controld...                                [  OK  ]
   Unfencing self...                                       [  OK  ]
   Joining fence domain...                                 [  OK  ]
[root@node3-f15 ~]# clustat
Cluster Status for cluster-f15 @ Wed Sep 28 14:39:17 2016
Member Status: Quorate


 Member Name                                             ID   Status
 ------ ----                                             ---- ------
 node1-f15.example.com                                       1 Online
 node2-f15.example.com                                       2 Online
 node3-f15.example.com                                       3 Online, Local


[root@node3-f15 ~]# service rgmanager start #启动rgmanager
Starting Cluster Service Manager:                          [  OK  ]
[root@node3-f15 ~]# clustat
Cluster Status for cluster-f15 @ Wed Sep 28 14:40:09 2016
Member Status: Quorate


 Member Name                             ID   Status
 ------ ----                             ---- ------
 node1-f15.example.com                       1 Online, rgmanager
 node2-f15.example.com                       2 Online, rgmanager
 node3-f15.example.com                       3 Online, Local, rgmanager


 Service Name                   Owner (Last)                   State         
 ------- ----                   ----- ------                   -----         
 service:myweb                  node2-f15.example.com          started       
[root@node3-f15 ~]# echo node3-f15 > /var/www/html/index.html#建立一个网页测试页
[root@node3-f15 ~]# ls /etc/cluster/fence_xvm.key #从主机拷贝key文件
/etc/cluster/fence_xvm.key
[root@node3-f15 ~]# ll /etc/cluster/fence_xvm.key #文件要保证大小一致才能使fence生效
-rw-r----- 1 root root 512 Sep 28 14:43 /etc/cluster/fence_xvm.key
[root@node3-f15 ~]# service network stop #停止后看服务是否切换
Shutting down interface eth0:  Write failed: Broken pipe
[kiosk@foundation15 Desktop]$ 
[kiosk@foundation15 Desktop]$ ssh root@172.25.15.12
Last login: Wed Sep 28 14:04:15 2016 from 172.25.15.250
[root@node3-f15 ~]# clustat
Could not connect to CMAN: No such file or directory
[root@node3-f15 ~]# service cman start #重启后要启动服务，或者加入到开机启动chkconfig modclusterd ricci cman rgmanager on
Starting cluster: 
   Checking if cluster has been disabled at boot...        [  OK  ]
   Checking Network Manager...                             [  OK  ]
   Global setup...                                         [  OK  ]
   Loading kernel modules...                               [  OK  ]
   Mounting configfs...                                    [  OK  ]
   Starting cman...                                        [  OK  ]
   Waiting for quorum...                                   [  OK  ]
   Starting fenced...                                      [  OK  ]
   Starting dlm_controld...                                [  OK  ]
   Tuning DLM kernel config...                             [  OK  ]
   Starting gfs_controld...                                [  OK  ]
   Unfencing self...                                       [  OK  ]
   Joining fence domain...                                 [  OK  ]
[root@node3-f15 ~]# clustat
Cluster Status for cluster-f15 @ Wed Sep 28 14:48:22 2016
Member Status: Quorate


 Member Name                             ID   Status
 ------ ----                             ---- ------
 node1-f15.example.com                       1 Online
 node2-f15.example.com                       2 Online
 node3-f15.example.com                       3 Online, Local


[root@node3-f15 ~]# service rgmanager start
Starting Cluster Service Manager:                          [  OK  ]
[root@node3-f15 ~]# clustat
Cluster Status for cluster-f15 @ Wed Sep 28 14:48:59 2016
Member Status: Quorate


 Member Name                                             ID   Status
 ------ ----                                             ---- ------
 node1-f15.example.com                                       1 Online
 node2-f15.example.com                                       2 Online
 node3-f15.example.com                                       3 Online, Local


[root@node3-f15 ~]# clustat #查看切换到了其他主机
\Cluster Status for cluster-f15 @ Wed Sep 28 14:49:34 2016
Member Status: Quorate


 Member Name                                             ID   Status
 ------ ----                                             ---- ------
 node1-f15.example.com                                       1 Online, rgmanager
 node2-f15.example.com                                       2 Online, rgmanager
 node3-f15.example.com                                       3 Online, Local, rgmanager


 Service Name                                   Owner (Last)                                   State         
 ------- ----                                   ----- ------                                   -----         
 service:myweb                                  node1-f15.example.com                          started 




[root@node2-f15 ~]# clusvcadm -r myweb -m node3-f15.example.com#可用这个命令直接切换服务的主机，不需要通过fence
Trying to relocate service:myweb to node3-f15.example.com...Success
service:myweb is now running on node3-f15.example.com


*************************************************************
[root@node1-f15 ~]# rm -f /var/lib/luci/data/luci.db #要想在web上可以管理node3，要先删除这个文件
[root@node1-f15 ~]# service luci restart #删除后重启服务
Stop luci...                                               [  OK  ]
Start luci...                                              [  OK  ]
Point your web browser to https://node1-f15.example.com:8084 (or equivalent) to access luci
***********************************************************
[kiosk@foundation15 Desktop]$ rht-vmctl start node4#开启node4作fdisk，仲裁盘
Starting node4.
[kiosk@foundation15 Desktop]$ ssh root@172.25.15.13


[root@rhel6 ~]# wget http://classroom.example.com/content/ula/cluster/clusterinit.sh


[root@rhel6 ~]# bash clusterinit.sh node4-f15.example.com#初始化环境
Stopping NetworkManager daemon:                            [  OK  ]
iptables: Saving firewall rules to /etc/sysconfig/iptables:[  OK  ]
[root@rhel6 ~]# reboot #重启


Broadcast message from [e-mail]
(/dev/pts/0) at 16:31 ...
The system is going down for reboot NOW!
[root@rhel6 ~]# Connection to 172.25.15.13 closed by remote host.
Connection to 172.25.15.13 closed.
[kiosk@foundation15 Desktop]$ ssh root@172.25.15.13
Last login: Wed Sep 28 16:28:35 2016 from 172.25.15.250
[kiosk@foundation15 Desktop]$ ssh root@172.25.15.13
ssh: connect to host 172.25.15.13 port 22: Connection refused
[kiosk@foundation15 Desktop]$ ssh root@172.25.15.13
Last login: Wed Sep 28 16:28:35 2016 from 172.25.15.250
[root@node4-f15 ~]# fdisk -l /dev/vdb


Disk /dev/vdb: 10.7 GB, 10737418240 bytes
16 heads, 63 sectors/track, 20805 cylinders
Units = cylinders of 1008 * 512 = 516096 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000


[root@node4-f15 ~]# fdisk /dev/vdb #新建磁盘


Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-20805, default 1): 
Using default value 1
Last cylinder, +cylinders or +size{K,M,G} (1-20805, default 20805): +64M#制定大小为64兆


Command (m for help): w
The partition table has been altered!ls /dev/vdb1


Calling ioctl() to re-read partition table.
Syncing disks.
[root@node4-f15 ~]# ls /dev/vdb1
/dev/vdb1
[root@node4-f15 ~]# yum install scsi* -y #安装scsi*的软件
[root@node4-f15 ~]# vim /etc/tgt/targets.conf #修改第38行
<target iqn.2008-09.com.example:server.target1>
    backing-store /dev/vdb1
</target>
[root@node4-f15 ~]# service tgtd start
Starting SCSI target daemon:                               [  OK  ]
***************node1  node2 node3:************
[root@node1-f15 ~]# yum install iscsi* -y
[root@node2-f15 ~]#  yum install iscsi* -y
[root@node3-f15 ~]#  yum install iscsi* -y
[root@node1-f15 ~]# iscsiadm -m discovery -t st -p 192.168.122.43
Starting iscsid:                                           [  OK  ]
192.168.122.43:3260,1 iqn.2008-09.com.example:server.target1
[root@node1-f15 ~]# iscsiadm -m node -l
Logging in to [iface: default, target: iqn.2008-09.com.example:server.target1, portal: 192.168.122.43,3260] (multiple)
Login to [iface: default, target: iqn.2008-09.com.example:server.target1, portal: 192.168.122.43,3260] successful.
[root@node2-f15 ~]# iscsiadm -m discovery -t st -p 192.168.122.43
Starting iscsid:                                           [  OK  ]
192.168.122.43:3260,1 iqn.2008-09.com.example:server.target1
[root@node2-f15 ~]# iscsiadm -m node -l
Logging in to [iface: default, target: iqn.2008-09.com.example:server.target1, portal: 192.168.122.43,3260] (multiple)
Login to [iface: default, target: iqn.2008-09.com.example:server.target1, portal: 192.168.122.43,3260] successful.
[root@node3-f15 ~]# iscsiadm -m discovery -t st -p 192.168.122.43
Starting iscsid:                                           [  OK  ]
192.168.122.43:3260,1 iqn.2008-09.com.example:server.target1
[root@node3-f15 ~]# iscsiadm -m node -l
Logging in to [iface: default, target: iqn.2008-09.com.example:server.target1, portal: 192.168.122.43,3260] (multiple)
Login to [iface: default, target: iqn.2008-09.com.example:server.target1, portal: 192.168.122.43,3260] successful.
[root@node1-f15 ~]# fdisk -l /dev/sda


Disk /dev/sda: 67 MB, 67576320 bytes
3 heads, 43 sectors/track, 1023 cylinders
Units = cylinders of 129 * 512 = 66048 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
[root@node1-f15 ~]# mkqdisk -c /dev/sda -l myqdisk #格式化指定名字为myqdisk
mkqdisk v3.0.12.1


Writing new quorum disk label 'myqdisk' to /dev/sda.
WARNING: About to destroy all data on /dev/sda; proceed [N/y] ? y


在web上配置
测试：
[root@node1-f15 ~]# clustat
[root@node2-f15 ~]# clustat
[root@node3-f15 ~]# clustat
都显示
 /dev/block/8:0                              0 Online, Quorum Disk


 Service Name                   Owner (Last)                   State         
 ------- ----                   ----- ------                   -----         
 service:myweb                  node3-f15.example.com          started   


[root@node3-f15 ~]# cman_tool status
Version: 6.2.0
Config Version: 19
Cluster Name: cluster-f15
Cluster Id: 34260
Cluster Member: Yes
Cluster Generation: 12
Membership state: Cluster-Member
Nodes: 3
Expected votes: 5
Quorum device votes: 2
Total votes: 5
Node votes: 1
Quorum: 3  
Active subsystems: 10
Flags: 
Ports Bound: 0 177 178  
Node name: node3-f15.example.com
Node ID: 3
Multicast addresses: 239.192.133.90 
Node addresses: 172.25.15.12 


*************************
共享存储
[root@node4-f15 ~]# mkdir /var/nfs/html -p
[root@node4-f15 ~]# echo cluster-f15 >/var/nfs/html/index.html
[root@node4-f15 ~]# vim /etc/exports 
[root@node4-f15 ~]# cat /etc/exports 
/var/nfs/html 192.168.122.0/24(rw,async)
[root@node4-f15 ~]# service nfs start
Starting NFS services:                                     [  OK  ]
Starting NFS quotas:                                       [  OK  ]
Starting NFS mountd:                                       [  OK  ]
Starting NFS daemon:                                       [  OK  ]
Starting RPC idmapd:                                       [  OK  ]
在web中将nfs以资源的方式加入，实现谁提供服务谁就去挂载
[root@node3-f15 ~]# clustat
Cluster Status for cluster-f15 @ Wed Sep 28 20:06:43 2016
Member Status: Quorate


 Member Name                             ID   Status
 ------ ----                             ---- ------
 node1-f15.example.com                       1 Online, rgmanager
 node2-f15.example.com                       2 Online, rgmanager
 node3-f15.example.com                       3 Online, Local, rgmanager
 /dev/block/8:0                              0 Online, Quorum Disk


 Service Name                   Owner (Last)                   State         
 ------- ----                   ----- ------                   -----         
 service:myweb                  node3-f15.example.com          started       
[root@node3-f15 ~]#  mount |grep html
192.168.122.43:/var/nfs/html on /var/www/html type nfs (rw,sync,soft,noac,vers=4,addr=192.168.122.43,clientaddr=192.168.122.42)
浏览器访问http://172.25.15.100/ 显示
cluster-f15