Keepalived实现Nginx反代的高可用

Keepalived实现Nginx反代的高可用

---

---

实验前的准备工作

本次实验使用四台centos7主机来搭建实验环境，实现Nginx的高可用解决方案。由于集群环境对时间要求严格，所以在实验开始前请务必先在四台主机上完成以下四项准备工作，特别是时间的同步。

具体的网络拓扑结构请看下图：

时间同步

root&proxy1: ~#yum -y install ntpdate ntproot&proxy1: ~#ntpdate 172.18.0.1  #和时钟服务器同步时间30 Oct 10:10:10 ntpdate[2148]: step time server 172.18.0.1 offset -28792.799544 sec#只有时间同步完成后，才能进行此步骤；root&proxy1: ~#vim /etc/chrony.conf# These servers were defined in the installation:server 172.18.0.1 iburst    #绑定时钟服务器，时钟服务器也可以来自网络，谷歌就可查到；server 0.centos.pool.ntp.org iburstserver 1.centos.pool.ntp.org iburst        …………此处省略部分显示内容…………root&proxy1: ~#systemctl enable chronydroot&proxy1: ~#systemctl start chronyd

关闭防火墙和SELinux

CentOS 7关闭防火墙的方法

root&proxy1: ~#systemctl disable firewalld.serviceroot&proxy1: ~#systemctl stop firewalld.serviceroot&proxy1: ~#iptables -F

关闭SElinux策略

root&proxy1: ~# sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/configroot&proxy1: ~# grep SELINUX=disabled /etc/selinux/config#确定SELINUX=disabled，已经将SElinux策略禁用SELINUX=disabledroot&proxy1: ~# setenforce 0#设置SELinux当前状态为Permissiveroot&proxy1: ~# getenforce#获取SELinux当前状态Permissive

做名称解析

root&proxy1: ~#hostnamectl set-hostname proxy1 #设置主机名的命令，执行exit命令后再进入生效root&proxy1: ~#vim /etc/hosts127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4::1         localhost localhost.localdomain localhost6 localhost6.localdomain6172.18.255.254  proxy1172.18.172.18   proxy2172.18.73.33    web1172.18.73.11    web2root&proxy1: ~#ping proxy2 #完成名称解析后可以直接通过主机名ping通对方root&proxy1: ~#ping web1root&proxy1: ~#ping web2

实现基于key的认证

root&proxy1: ~#cd ~root&proxy1: ~#ssh-keygen  #生成秘钥的命令。root&proxy1: ~#cd .sshroot&proxy1: .ssh#ssh-copy-id -i id_rsa.pub proxy2: #将公钥文件发送给proxy2；root&proxy1: .ssh#ssh proxy2Last login: Mon Oct 30 18:07:57 2017 from 172.18.10.43root&proxy2: ~#    #完成基于key的认证后可以不需要输入密码，直接连接对方主机

---

主代理proxy1的配置

注意：运行Keepalived服务需要网卡支持多播；ip a 会有MULTICAST字样

安装所需的软件包

root&proxy1: ~#yum -y install keepalived nginx psmisc mailx

定义通知脚本

此脚本的作用是在Keepalived服务发生故障时，以邮件的方式通知管理员。

root&proxy1: ~#vim /etc/keepalived/notify.sh#!/bin/bashcontact='root@localhost'notify() {        mailsubject="$(hostname) to be $1, vip floating"        mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"        echo "$mailbody" | mail -s "$mailsubject" $contact}case $1 inmaster)        notify master        ;;backup)        notify backup        ;;fault)        notify fault        ;;*)        echo "Usage: $(basename $0) {master|backup|fault}"        exit 1        ;;esacroot&proxy1: ~#chmod +x /etc/keepalived/notify.shroot&proxy1: ~#scp /etc/keepalived/notify.sh proxy2:/etc/keepalived/

定义Keepalived的日志存储

定义Keepalived的日志存储，有利于在发生故障后为排错提供参考数据；

root&proxy1: ~#vim /etc/sysconfig/keepalivedKEEPALIVED_OPTIONS="-D -S 2"root&proxy1: ~#vim /etc/rsyslog.conflocal2.*                                             /var/log/keepalived.logroot&proxy1: ~#systemctl restart rsyslogroot&proxy1: ~#systemctl restart keepalived

配置Keepalived服务

Keepalived服务是一个基于vrrp协议的服务，主要用于实现集群的高可用性，更多关于vrrp协议的知识可以查看《vrrp技术白皮书》。

root&proxy1: ~#vim /etc/keepalived/keepalived.conf! Configuration File for keepalivedglobal_defs {       #全局配置   notification_email { #定义收邮件的邮箱     root@localhost     [e-mail]   }   notification_email_from proxy1@localhost #定义发邮件的邮箱   smtp_server 127.0.0.1    #定义邮件服务器的地址   smtp_connect_timeout 30  #邮件的超时时长   router_id proxy1     #ID对应主机名   vrrp_mcast_group4    224.111.111.111 #定义vrrp协议使用的多播地址}vrrp_script chk_down {  #用于控制Keepalived优先级的脚本    script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"    interval 1    #通过每秒检测一次/etc/keepalived/down文件是否存在，来控制服务优先级。    weight -20    #文件存在优先级-20，主要用于服务调试。}vrrp_script chk_nginx { #检查Nginx反代存活状态的脚本        script "killall -0 nginx &> /dev/null && exit 0 || exit 1"  #yum install psmisc -y        interval 1        weight -20   #定时通过killall命令发送0信号检测Nginx服务是否还存活。        fall 3   #检测到三次服务不存活就主动下调优先级        rise 3   #检测到三次服务存活就主动恢复优先级}vrrp_instance VI_1 {    #虚拟路由器的配置    state MASTER   #此处定义了设备的主备关系，此处为MASTER，另一台主机的此处就需设为BACKUP。    interface ens32   #对外接口的网卡名    virtual_router_id 88    #定义虚拟路由器的编号，同一组设备需要编号一致。    priority 100    #优先级编号，主设备的优先级要高于从设备；    advert_int 1    #发送公告的间隔时间；1秒    authentication {        auth_type PASS        auth_pass 8poTWQ==  #主备通信的加密口令；八位以内随机数；    }    virtual_ipaddress {     #定义VIP的地址，此地址会自动绑定在优先级高的设备上，接收用户的请求；        172.18.22.22/16    }    #以下几项是对上面脚本的调用；    notify_master   "/etc/keepalived/notify.sh master"    notify_backup   "/etc/keepalived/notify.sh backup"    notify_fault    "/etc/keepalived/notify.sh fault"    track_script {        chk_down    #调用chk_down脚本        chk_nginx   #调用chk_nginx脚本    }}root&proxy1: ~#systemctl start keepalived  #启动Keepalived服务

配置Nginx的反向代理

root&proxy1: ~#vim /etc/nginx/nginx.confhttp {        ……此处省略部分显示内容……    upstream websrvs {              #需定义于http字段        server 172.18.73.33:80 weight=1;   #后端的被代理服务器地址，以及调度的权重。        server 172.18.73.11:80 weight=1;        server 127.0.0.1:8000 backup;   #sorry_server的地址，用于后端服务器都挂掉时。     }        ……此处省略部分显示内容……}server {        listen 127.0.0.1:8000 ;    #定义sorry_server监听的端口以及主目录所在路径；        root         /usr/share/nginx/html;}server {        ……此处省略部分显示内容……    location / {    #需要定义于server字段        proxy_pass http://websrvs;     #访问本机web服务的"/"时向后调度的组    }                ……此处省略部分显示内容……}root&proxy1: ~#nginx -troot&proxy1: ~#echo sorry_server1 > /usr/share/nginx/html/index.htmlroot&proxy1: ~#nginx

---

备用代理proxy2的配置

安装所需的软件包

root&proxy2: ~#yum -y install keepalived nginx psmisc mailx

定义Keepalived的日志存储

root&proxy2: ~#vim /etc/sysconfig/keepalivedKEEPALIVED_OPTIONS="-D -S 2"root&proxy2: ~#vim /etc/rsyslog.conflocal2.*                                           /var/log/keepalived.logroot&proxy2: ~#systemctl restart rsyslogroot&proxy2: ~#systemctl restart keepalived

配置Keepalived服务

root&proxy1: ~#vim /etc/keepalived/keepalived.conf! Configuration File for keepalivedglobal_defs {          notification_email {     root@localhost     [e-mail]   }   notification_email_from proxy2@localhost    smtp_server 127.0.0.1       smtp_connect_timeout 30     router_id proxy2     #ID对应主机名   vrrp_mcast_group4    224.111.111.111 #两个设备多播地址要相同；}vrrp_script chk_down {      script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"        interval 1        weight -20}vrrp_script chk_nginx {     script "killall -0 nginx &> /dev/null && exit 0 || exit 1"          interval 1        weight -20        fall 3        rise 3}vrrp_instance VI_1 {    state BACKUP      #此设备为BACKUP     interface ens32        virtual_router_id 88    #定义虚拟路由器的编号    priority 90     #优先级编号，要低于MASTERT设备；    advert_int 1        authentication {        auth_type PASS        auth_pass 8poTWQ==  #两个设备要口令相同    }    virtual_ipaddress {     #定义VIP的地址        172.18.22.22/16    }    notify_master   "/etc/keepalived/notify.sh master"    notify_backup   "/etc/keepalived/notify.sh backup"    notify_fault    "/etc/keepalived/notify.sh fault"    track_script {        chk_down            chk_nginx    }}root&proxy2: ~#systemctl start keepalived

配置备用的Nginx反向代理

root&proxy2: ~#vim /etc/nginx/nginx.confhttp {        …………    upstream websrvs {          server 172.18.73.33:80 weight=1;        server 172.18.73.11:80 weight=1;        server 127.0.0.1:8000 backup;    }        …………}server {        listen 127.0.0.1:8000 ;        root         /usr/share/nginx/html;}server {        …………    location / {            proxy_pass http://websrvs;    }                    …………}root&proxy2: ~#nginx -troot&proxy2: ~#echo sorry_server2 > /usr/share/nginx/html/index.htmlroot&proxy2: ~#nginx

---

后端被代理服务器web1的配置

root&web1: ~#yum -y install nginxroot&web1: ~#echo nginx web1 > /usr/share/nginx/html/index.htmlroot&web1: ~#nginx

后端被代理服务器web2的配置

root&web2: ~#yum -y install nginxroot&web2: ~#echo nginx web2 > /usr/share/nginx/html/index.htmlroot&web2: ~#nginx

---

测试高可用性

后端web服务器单点失败

停掉web1的Nginx服务，代理服务器可以自动检测到web1宕掉，不再向坏掉的web1调度数据包。

root&web1: ~#nginx -s stop

root&client: ~#for i in {1..1000} ; do sleep 0.5 ;curl 172.18.22.22 ; donenginx web2nginx web2nginx web2nginx web2

后端web服务器全部失败

停掉web1和web2两台主机的Nginx服务，前端代理服务器会自动响应sorry server；

root&web1: ~#nginx -s stop

root&web2: ~#nginx -s stop

root&client: ~#for i in {1..1000} ; do sleep 0.5 ;curl 172.18.22.22 ; donesorry_server1sorry_server1sorry_server1

Nginx代理导致的前端代理单点失败

宕掉前端主代理服务器的Nginx服务，备用代理服务器会主动接过VIP响应客户端的请求，但会有几秒钟不能响应客户端请求。

root&proxy1: ~#nginx -s stop

root&client: ~#for i in {1..1000} ; do sleep 0.5 ;curl 172.18.22.22 ; donenginx web2nginx web1nginx web2nginx web1curl: (7) couldn't connect to hostcurl: (7) couldn't connect to hostcurl: (7) couldn't connect to hostcurl: (7) couldn't connect to hostcurl: (7) couldn't connect to hostcurl: (7) couldn't connect to hostcurl: (7) couldn't connect to hostnginx web2nginx web2nginx web1

Keepalived导致的前端代理单点失败

宕掉前端主代理服务器的Keepalived服务，备用代理服务器会主动接过VIP响应客户端的请求，请求不会中断。

root&proxy1: ~#systemctl stop keepalived

root&client: ~#for i in {1..1000} ; do sleep 0.5 ;curl 172.18.22.22 ; donenginx web1nginx web2nginx web1nginx web2nginx web1nginx web2nginx web1

观察VIP的迁移
服务宕掉前


服务宕掉后