手动部署 ceph mon (luminous 版)

环境

参考 手动部署 ceph 环境说明 (luminous 版)

create ceph.conf.template

[global]fsid = $cephuuidmon initial members = $cephmonhostmon host = $cephmonipaddrpublic network = $cephmonnetworkcluster network = $cephdatanetworkauth cluster required = cephxauth service required = cephxauth client required = cephxosd journal size = 2048filestore xattr use omap = trueosd pool default size = 3osd pool default min size = 1osd pool default pg num = $cephpgnumosd pool default pgp num = $cephpgpnumosd crush chooseleaf type = 1[osd]osd journal = /var/lib/ceph/journal/ceph-\$id/journal    <- 这个变量符号需要保留, 因此 \$osd journal size = 2048

生成 ceph.conf

[root@hh-ceph-128040 ceph]# cat gen_ceph_conf.sh#!/bin/bashsource parametereval "cat <<EOF$(<ceph.conf.template)EOF" 2> /dev/null

效果

[root@hh-ceph-128040 ceph]# . gen_ceph_conf.sh[global]fsid = c45b752d-5d4d-4d3a-a3b2-04e73eff4ccdmon initial members = hh-ceph-128040,hh-ceph-128214,hh-ceph-128215mon host = 10.199.128.40,10.199.128.214,10.199.128.215public network = 10.199.128.0/22cluster network = 240.30.128.0/21auth cluster required = cephxauth service required = cephxauth client required = cephxosd journal size = 2048filestore xattr use omap = trueosd pool default size = 3osd pool default min size = 1osd pool default pg num = 1024osd pool default pgp num = 1024osd crush chooseleaf type = 1[osd]osd journal = /var/lib/ceph/journal/ceph-$id/journalosd journal size = 2048

说明: 注意, 这里使用到独立的磁盘作为 journal 存放,因此自行修改一下

创建 mon keyring

[root@hh-ceph-128040 ceph]# ceph-authtool --create-keyring /etc/ceph/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'creating /etc/ceph/ceph.mon.keyring[root@hh-ceph-128040 ceph]# cat /etc/ceph/ceph.mon.keyring[mon.]        key = AQD3ug5a3jQiGRAAUfiSG6LKPmubrW5a7E79dg==        caps mon = "allow *"

创建 admin keyring

[root@hh-ceph-128040 ceph]# ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'creating /etc/ceph/ceph.client.admin.keyring[root@hh-ceph-128040 ceph]# cat /etc/ceph/ceph.client.admin.keyring[client.admin]        key = AQA3uw5ap6cxOxAAFzfyucOfOjmp7nwu6c+/UA==        auid = 0        caps mds = "allow"        caps mon = "allow *"        caps osd = "allow *"

导入 admin keyring

注入 admin keyring 前

[root@hh-ceph-128040 ceph]# cat /etc/ceph/ceph.mon.keyring[mon.]        key = AQD3ug5a3jQiGRAAUfiSG6LKPmubrW5a7E79dg==        caps mon = "allow *"

执行注入 admin keyring

[root@hh-ceph-128040 ceph]# ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyringimporting contents of /etc/ceph/ceph.client.admin.keyring into /etc/ceph/ceph.mon.keyring

注入 admin keyring 后

[root@hh-ceph-128040 ceph]# cat /etc/ceph/ceph.mon.keyring[mon.]        key = AQD3ug5a3jQiGRAAUfiSG6LKPmubrW5a7E79dg==        caps mon = "allow *"[client.admin]        key = AQA3uw5ap6cxOxAAFzfyucOfOjmp7nwu6c+/UA==        auid = 0        caps mds = "allow"        caps mon = "allow *"        caps osd = "allow *"

注册其他的 mon

[root@hh-ceph-128040 ceph]# monmaptool --create --add  hh-ceph-128040 10.199.128.40 --add hh-ceph-128214 10.199.128.214  --add hh-ceph-128215 10.199.128.215 --fsid c45b752d-5d4d-4d3a-a3b2-04e73eff4ccd  /etc/ceph/monmapmonmaptool: monmap file /etc/ceph/monmapmonmaptool: set fsid to c45b752d-5d4d-4d3a-a3b2-04e73eff4ccdmonmaptool: writing epoch 0 to /etc/ceph/monmap (3 monitors)

下创建对应 mon 目录

[root@hh-ceph-128040 ~]#  mkdir -p /var/lib/ceph/mon/ceph-hh-ceph-128040[root@hh-ceph-128214 ~]#  mkdir -p /var/lib/ceph/mon/ceph-hh-ceph-128214[root@hh-ceph-128215 ~]#  mkdir -p /var/lib/ceph/mon/ceph-hh-ceph-128215

复制

复制所有的 keyring ceph.conf monmap 文件到另外两台主机中

[root@hh-ceph-128040 ceph]# scp /etc/ceph/* 10.199.128.214:/etc/ceph/The authenticity of host '10.199.128.214 (10.199.128.214)' can't be established.ECDSA key fingerprint is 8b:31:2f:c4:ad:5c:b0:0f:19:ef:32:bf:89:5c:10:b9.Are you sure you want to continue connecting (yes/no)? yesWarning: Permanently added '10.199.128.214' (ECDSA) to the list of known hosts.root@10.199.128.214's password:ceph.client.admin.keyring                                                                                                                  100%  137     0.1KB/s   00:00ceph.conf                                                                                                                                  100%  540     0.5KB/s   00:00ceph.mon.keyring                                                                                                                           100%  214     0.2KB/s   00:00monmap                                                                                                                                     100%  474     0.5KB/s   00:00rbdmap                                                                                                                                     100%   92     0.1KB/s   00:00[root@hh-ceph-128040 ceph]# scp /etc/ceph/* 10.199.128.215:/etc/ceph/The authenticity of host '10.199.128.215 (10.199.128.215)' can't be established.ECDSA key fingerprint is da:99:7f:ae:10:21:d7:60:8e:04:71:d4:11:80:89:d5.Are you sure you want to continue connecting (yes/no)? yesWarning: Permanently added '10.199.128.215' (ECDSA) to the list of known hosts.root@10.199.128.215's password:ceph.client.admin.keyring                                                                                                                  100%  137     0.1KB/s   00:00ceph.conf                                                                                                                                  100%  540     0.5KB/s   00:00ceph.mon.keyring                                                                                                                           100%  214     0.2KB/s   00:00monmap                                                                                                                                     100%  474     0.5KB/s   00:00rbdmap                                                                                                                                     100%   92     0.1KB/s   00:00

初始化 mon db

[root@hh-ceph-128040 ~]# ceph-mon --mkfs -i hh-ceph-128040 --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring[root@hh-ceph-128214 ~]# ceph-mon --mkfs -i hh-ceph-128214 --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring[root@hh-ceph-128215 ~]# ceph-mon --mkfs -i hh-ceph-128215 --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring

后续

创建允许服务启动的空文件

[root@hh-ceph-128040 ~]# touch /var/lib/ceph/mon/ceph-hh-ceph-128040/{done,upstart}[root@hh-ceph-128214 ~]# touch /var/lib/ceph/mon/ceph-hh-ceph-128214/{done,upstart}[root@hh-ceph-128215 ~]# touch /var/lib/ceph/mon/ceph-hh-ceph-128215/{done,upstart}

启动服务文件

[root@hh-ceph-128040 /]# cp /usr/lib/systemd/system/ceph-mon@.service   /usr/lib/systemd/system/[e-mail][root@hh-ceph-128214 /]# cp /usr/lib/systemd/system/ceph-mon@.service   /usr/lib/systemd/system/[e-mail][root@hh-ceph-128215 /]# cp /usr/lib/systemd/system/ceph-mon@.service   /usr/lib/systemd/system/[e-mail]

管理服务与文件属性

systemctl daemon-reloadchown -R ceph:ceph /var/log/ceph  /var/lig/ceph/mon

启动 mon

[root@hh-ceph-128040 ceph]# systemctl start ceph-mon@hh-ceph-128040[root@hh-ceph-128040 ceph]# systemctl status ceph-mon@hh-ceph-128040● [e-mail] - Ceph cluster monitor daemon   Loaded: loaded (/usr/lib/systemd/system/[e-mail]; disabled; vendor preset: disabled)   Active: active (running) since 五 2017-11-17 21:15:08 CST; 1s ago Main PID: 3429 (ceph-mon)   CGroup: /system.slice/system-ceph\x2dmon.slice/[e-mail]           └─3429 /usr/bin/ceph-mon -f --cluster ceph --id hh-ceph-128040 --setuser ceph --setgroup ceph11月 17 21:15:08 hh-ceph-128040.vclound.com systemd[1]: Started Ceph cluster monitor daemon.11月 17 21:15:08 hh-ceph-128040.vclound.com systemd[1]: Starting Ceph cluster moni

检测

[root@hh-ceph-128040 ceph]# ceph -s  cluster:    id:     c45b752d-5d4d-4d3a-a3b2-04e73eff4ccd    health: HEALTH_OK  services:    mon: 3 daemons, quorum hh-ceph-128040,hh-ceph-128214,hh-ceph-128215    mgr: no daemons active    osd: 0 osds: 0 up, 0 in  data:    pools:   0 pools, 0 pgs    objects: 0 objects, 0 bytes    usage:   0 kB used, 0 kB / 0 kB avail    pgs: