Ceph安装对象网关

1.概述
安装3个网关节点分别是：controller-03，controller-04，controller-05，使用ceph gw自带的Civetweb，前端使用nginx作为前端代理。

2.分别在三个节点安装ceph gw
分别在每个gw节点执行：

yum install ceph-radosgw

然后再执行：

ceph-deploy --overwrite rgw create controller-03 controller-04 controller-05成功

可以看到网关已经运行

ceph rgw默认使用自带Civetweb提供服务，在浏览器输入ip:7480，可以看到正确运行

<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName/></Owner><Buckets/></ListAllMyBucketsResult>

3.配置各节点网关
在controller-03执行：ceph auth get client.rgw.controller-03 > /etc/ceph/ceph.client.radosgw.keyring
在controller-04执行：ceph auth get client.rgw.controller-04 > /etc/ceph/ceph.client.radosgw.keyring
在controller-05执行：ceph auth get client.rgw.controller-05 > /etc/ceph/ceph.client.radosgw.keyring

配置ceph.conf文件：[client.rgw.controller-03]rgw_frontends = "civetweb port=8899"host = controller-03keyring = /etc/ceph/ceph.client.radosgw.keyringrgw dns name = xiangcloudos.comrgw socket path = /var/run/ceph/ceph-client.rgw.controller-03.asokrgw content length compat = true[client.rgw.controller-04]rgw_frontends = "civetweb port=8899"host = controller-04keyring = /etc/ceph/ceph.client.radosgw.keyringrgw dns name = xiangcloudos.comrgw socket path = /var/run/ceph/ceph-client.rgw.controller-04.asokrgw content length compat = true[client.rgw.controller-05]rgw_frontends = "civetweb port=8899"host = controller-05keyring = /etc/ceph/ceph.client.radosgw.keyringrgw dns name = xiangcloudos.comrgw socket path = /var/run/ceph/ceph-client.rgw.controller-05.asokrgw content length compat = true

分发配置然后重新启动每个节点的cephgw,执行 service radosgw restart.

4.nginx配置
当前在controller-01上安装nginx，配置nginx作为cephgw的前端代理。

http {log_format main '$remote_addr - $remote_user [$time_local] "$request" ''$status $body_bytes_sent "$http_referer" ''"$http_user_agent" "$http_x_forwarded_for"';access_log /var/log/nginx/access.log main;sendfile on;tcp_nopush on;tcp_nodelay on;keepalive_timeout 65;types_hash_max_size 2048;include /etc/nginx/mime.types;default_type application/octet-stream;# Load modular configuration files from the /etc/nginx/conf.d directory.# See http://nginx.org/en/docs/ngx_core_module.html#include# for more information.include /etc/nginx/conf.d/*.conf;upstream ceph_radosgw_zone {　　server controller-03:8899 weight=1 max_fails=2 fail_timeout=5;　　server controller-04:8899 weight=1 max_fails=2 fail_timeout=5;　　server controller-05:8899 weight=1 max_fails=2 fail_timeout=5;}server {listen 81 default_server;listen [::]:81 default_server;server_name xiangcloudos.com;# Load configuration files for the default server block.include /etc/nginx/default.d/*.conf;if ($host ~* (.*)\.xiangcloudos\.com$){set $sub_dom '$1';rewrite ^(.*)$ /$sub_dom$1 last;}location /{proxy_pass http://ceph_radosgw_zone;}}}

5.创建ceph rgw存储池

ceph osd pool create .rgw 128 128 replicated_sata_rulesetceph osd pool create .rgw.root 128 128 replicated_sata_rulesetceph osd pool create .rgw.control 128 128 replicated_sata_rulesetceph osd pool create .rgw.gc 128 128 replicated_sata_rulesetceph osd pool create .rgw.buckets 128 128 replicated_sata_rulesetceph osd pool create .rgw.buckets.index 128 128 replicated_sata_rulesetceph osd pool create .rgw.buckets.extra 128 128 replicated_sata_rulesetceph osd pool create .log 128 128 replicated_sata_rulesetceph osd pool create .intent-log 128 128 replicated_sata_rulesetceph osd pool create .usage 128 128 replicated_sata_rulesetceph osd pool create .users 128 128 replicated_sata_rulesetceph osd pool create .users.email 128 128 replicated_sata_rulesetceph osd pool create .users.swift 128 128 replicated_sata_rulesetceph osd pool create .users.uid 128 128 replicated_sata_ruleset

6.创建radosgw用户进行访问

radosgw-admin user create --uid="admin" --display-name="admin"{　　"user_id": "admin",　　"display_name": "admin",　　"email": "",　　"suspended": 0,　　"max_buckets": 1000,　　"auid": 0,　　"subusers": [],　　"keys": [　　{　　　　"user": "admin",　　　　"access_key": "8NCG9CGJ1XHAUUXSJ5IS",　　　　"secret_key": "bC4POA9I8tmC1FgP1DCFx5YpB7m6kExzFkoaO0Hm"　　}　　],　　"swift_keys": [],　　"caps": [],　　"op_mask": "read, write, delete",　　"default_placement": "",　　"placement_tags": [],　　"bucket_quota": {　　　　"enabled": false,　　　　"max_size_kb": -1,　　　　"max_objects": -1},"user_quota": {　　"enabled": false,　　"max_size_kb": -1,　　"max_objects": -1},"temp_url_keys": []}创建swift用户：radosgw-admin subuser create --uid=admin --subuser=admin --access=full{　　"user_id": "admin",　　"display_name": "admin",　　"email": "",　　"suspended": 0,　　"max_buckets": 1000,　　"auid": 0,"subusers": [{　　"id": "admin:admin",　　"permissions": "full-control"}],"keys": [{　　"user": "admin",　　"access_key": "8NCG9CGJ1XHAUUXSJ5IS",　　"secret_key": "bC4POA9I8tmC1FgP1DCFx5YpB7m6kExzFkoaO0Hm"},{　　"user": "admin:admin",　　"access_key": "FI05ZE4AEV3H8Y0BJP6D",　　"secret_key": ""}],"swift_keys": [],"caps": [],"op_mask": "read, write, delete","default_placement": "","placement_tags": [],"bucket_quota": {　　"enabled": false,　　"max_size_kb": -1,　　"max_objects": -1},"user_quota": {　　"enabled": false,　　"max_size_kb": -1,　　"max_objects": -1},"temp_url_keys": []}radosgw-admin key create --subuser=admin:swift --key-type=swift --gen-secret{　　"user_id": "admin",　　"display_name": "admin",　　"email": "",　　"suspended": 0,　　"max_buckets": 1000,　　"auid": 0,　　"subusers": [　　{　　　　"id": "admin:admin",　　　　"permissions": "full-control"　　}　　],　　"keys": [　　{　　"user": "admin",　　"access_key": "8NCG9CGJ1XHAUUXSJ5IS",　　"secret_key": "bC4POA9I8tmC1FgP1DCFx5YpB7m6kExzFkoaO0Hm"　　},　　{　　"user": "admin:admin",　　"access_key": "FI05ZE4AEV3H8Y0BJP6D",　　"secret_key": ""　　}　　],　　"swift_keys": [　　{　　"user": "admin:swift",　　"secret_key": "2BQyi6uMi1lKfq1fDcrYiA2ApcUimtYzXMuiRoC8"　　}　　],　　"caps": [],　　"op_mask": "read, write, delete",　　"default_placement": "",　　"placement_tags": [],　　"bucket_quota": {　　"enabled": false,　　"max_size_kb": -1,　　"max_objects": -1　　},　　"user_quota": {　　"enabled": false,　　"max_size_kb": -1,　　"max_objects": -1　　},"temp_url_keys": []}

测试

radosgw-admin user create --uid="admin" --display-name="admin"{　　"user_id": "admin",　　"display_name": "admin",　　"email": "",　　"suspended": 0,　　"max_buckets": 1000,　　"auid": 0,　　"subusers": [],　　"keys": [　　{　　　　"user": "admin",　　　　"access_key": "8NCG9CGJ1XHAUUXSJ5IS",　　　　"secret_key": "bC4POA9I8tmC1FgP1DCFx5YpB7m6kExzFkoaO0Hm"　　}　　],　　"swift_keys": [],　　"caps": [],　　"op_mask": "read, write, delete",　　"default_placement": "",　　"placement_tags": [],　　"bucket_quota": {　　　　"enabled": false,　　　　"max_size_kb": -1,　　　　"max_objects": -1},"user_quota": {　　"enabled": false,　　"max_size_kb": -1,　　"max_objects": -1},"temp_url_keys": []}创建swift用户：radosgw-admin subuser create --uid=admin --subuser=admin --access=full{　　"user_id": "admin",　　"display_name": "admin",　　"email": "",　　"suspended": 0,　　"max_buckets": 1000,　　"auid": 0,"subusers": [{　　"id": "admin:admin",　　"permissions": "full-control"}],"keys": [{　　"user": "admin",　　"access_key": "8NCG9CGJ1XHAUUXSJ5IS",　　"secret_key": "bC4POA9I8tmC1FgP1DCFx5YpB7m6kExzFkoaO0Hm"},{　　"user": "admin:admin",　　"access_key": "FI05ZE4AEV3H8Y0BJP6D",　　"secret_key": ""}],"swift_keys": [],"caps": [],"op_mask": "read, write, delete","default_placement": "","placement_tags": [],"bucket_quota": {　　"enabled": false,　　"max_size_kb": -1,　　"max_objects": -1},"user_quota": {　　"enabled": false,　　"max_size_kb": -1,　　"max_objects": -1},"temp_url_keys": []}radosgw-admin key create --subuser=admin:swift --key-type=swift --gen-secret{　　"user_id": "admin",　　"display_name": "admin",　　"email": "",　　"suspended": 0,　　"max_buckets": 1000,　　"auid": 0,　　"subusers": [　　{　　　　"id": "admin:admin",　　　　"permissions": "full-control"　　}　　],　　"keys": [　　{　　"user": "admin",　　"access_key": "8NCG9CGJ1XHAUUXSJ5IS",　　"secret_key": "bC4POA9I8tmC1FgP1DCFx5YpB7m6kExzFkoaO0Hm"　　},　　{　　"user": "admin:admin",　　"access_key": "FI05ZE4AEV3H8Y0BJP6D",　　"secret_key": ""　　}　　],　　"swift_keys": [　　{　　"user": "admin:swift",　　"secret_key": "2BQyi6uMi1lKfq1fDcrYiA2ApcUimtYzXMuiRoC8"　　}　　],　　"caps": [],　　"op_mask": "read, write, delete",　　"default_placement": "",　　"placement_tags": [],　　"bucket_quota": {　　"enabled": false,　　"max_size_kb": -1,　　"max_objects": -1　　},　　"user_quota": {　　"enabled": false,　　"max_size_kb": -1,　　"max_objects": -1　　},"temp_url_keys": []}