lvs-fullnat + keepalived

来源：互联网发布：java培训好不好编辑：程序博客网时间：2024/05/22 17:47

本地到lvs官网(http://kb.linuxvirtualserver.org/wiki/IPVS_FULLNAT_and_SYNPROXY)下载操作手册：http://kb.linuxvirtualserver.org/images/c/c8/LVS%E6%93%8D%E4%BD%9C%E6%89%8B%E5%86%8C.zip

lvs-fullnat模式，需要在kernel里面打上lvs的补丁，才能支持lvs-fullnat，因此最好选择一个纯净的kernel版本进行内核编译，然后再打补丁，需要满足以下条件：

1.关掉iptables和selinux=disabled

2.支持lvs-fullnat的内核

3.支持lvs-fullnat(FNAT)模式的keepalived

4.支持full-nat的ipvsadm

因此需要到lvs官网下载lvs补丁包，kernel最好是纯净版，在红帽官网下载就行，lvs-fullnat的补丁包在http://kb.linuxvirtualserver.org/images/a/a5/Lvs-fullnat-synproxy.tar.gz，但是解压后会发现只有kernel-2.6.32.220版本的，前端的Director需要打lvs-2.6.32-220.23.1.el6.patch这个补丁，而realserver需要toa-2.6.32-220.23.1.el6.patch这个补丁，之前鄙人已经尝试过2.6.32-431,2.6.32-573版本的，发现如果431版本的kernel的Director在打kernel的lvs-2.6.32-220.23.1.el6.patch补丁时报错条目还是比较多的，如果要修复得花上不少时间，因此直接换成将Centos6.7的操作系统换成centos6.2了，Director的版本可以和realserver不一样，这样顺利多了，对于realserver（centos6.5）在打toa-2.6.32-220.23.1.el6.patch这个补丁时会报一些错，但都还比较容易修改，因此我的Director和Realserver的版本如下：

DS: centos6.2

kernel 版本： 2.6.32-220.el6.x86_64

lvs_dr01

eth0 ： 10.129.45.100

lvs_dr02:

eth0: 10.129.45.106

RS:centos6.5

kernel版本： 2.6.32-431.el6.x86_64

lvs_rs01:

eth0: 10.129.45.102

lvs_rs02:

eth0:10.129.45.103

lvs_rs03:

eth0:10.129.45.104

lvs_rs04:

eth0:10.129.45.105

Director:

wget http://kb.linuxvirtualserver.org/images/a/a5/Lvs-fullnat-synproxy.tar.gz

wget ftp://ftp.redhat.com/pub/redhat/linux/enterprise/6Server/en/os/SRPMS/kernel-2.6.32-573.26.1.el6.src.rpm

useradd lvs

groupadd mockbuild

useradd -g mockbuild mockbuild

echo '%_topdir /home/lvs/rpms' >> ~/.rpmmacros
echo '%_tmppath /home/lvs/rpms/tmp' >> ~/.rpmmacros
echo '%_sourcedir /home/lvs/rpms/SOURCES' >> ~/.rpmmacros
echo '%_specdir /home/lvs/rpms/SPECS' >> ~/.rpmmacros
echo '%_srcrpmdir /home/lvs/rpms/SRPMS' >> ~/.rpmmacros
echo '%_rpmdir /home/lvs/rpms/RPMS' >> ~/.rpmmacros
echo '%_builddir /home/lvs/rpms/BUILD' >> ~/.rpmmacros

cd /home/lvs/

mkdir rpms/{tmp,SOURCES,SPECS,SRPMS,RPMS,BUILD} -pv

rpm -ivh /usr/local/src/lvs/kernel-2.6.32-220.26.1.el6.src.rpm

yum install -y rpm-build-4.8.0-47.el6.x86_64 xmlto asciidoc elfutils-libelf-devel elfutils-devel zlib-devel binutils-devel newt-devel python-devel audit-libs-devel perl hmaccalc perl-ExtUtils-Embed rng-tools

##说明：rng-tools用于在执行rpmbuild -bb --target=`uname -m` kernel.spec的时候生成随机数，不然会卡在那里，但是根据卡的地方倒退回去会看到提示就执行rngd -r /dev/hwrandom，不行的话执行 rngd -r /dev/urandom，因此需要安装此工具

yum -y groupinstall "Development tools"

cd /usr/local/src/lvs/

tar xf Lvs-fullnat-synproxy.tar.gz

cd /home/lvs/rpms/SPECS/

rpmbuild -bp kernel.spec

cd /home/lvs/rpms/BUILD/

cd kernel-2.6.32-220.23.1.el6/linux-2.6.32-220.23.1.el6.x86_64/

cp /usr/local/src/lvs/lvs-fullnat-synproxy/lvs-2.6.32-220.23.1.el6.patch ./

patch -p1<lvs-2.6.32-220.23.1.el6.patch

make -j16;
make modules_install;
make install;

修改vim /home/lvs/rpms/SOURCES/config-generic下面的值为20，默认是12

sed -i 's/CONFIG_IP_VS_TAB_BITS=.*$/CONFIG_IP_VS_TAB_BITS=20/g' /home/lvs/rpms/SOURCES/config-generic

Realserer:

[root@lvs_rs02 lvs]# cat lvs_rs.sh
#!/bin/bash
useradd lvs
groupadd mockbuild
useradd -g mockbuild mockbuild
echo '%_topdir /home/lvs/rpms' >> ~/.rpmmacros
echo '%_tmppath /home/lvs/rpms/tmp' >> ~/.rpmmacros
echo '%_sourcedir /home/lvs/rpms/SOURCES' >> ~/.rpmmacros
echo '%_specdir /home/lvs/rpms/SPECS' >> ~/.rpmmacros
echo '%_srcrpmdir /home/lvs/rpms/SRPMS' >> ~/.rpmmacros
echo '%_rpmdir /home/lvs/rpms/RPMS' >> ~/.rpmmacros
echo '%_builddir /home/lvs/rpms/BUILD' >> ~/.rpmmacros

cd /home/lvs/
mkdir rpms/{tmp,SOURCES,SPECS,SRPMS,RPMS,BUILD} -pv
rpm -ivh /usr/local/src/lvs/kernel-2.6.32-431.el6.src.rpm

yum install -y rpm-build xmlto asciidoc elfutils-libelf-devel elfutils-devel zlib-devel binutils-devel newt-devel python-devel audit-libs-devel perl hmaccalc perl-ExtUtils-Embed rng-tools
yum -y groupinstall "Development tools"
cd /home/lvs/rpms/SPECS/
#rpmbuild -bb --target=`uname -m` kernel.spec
rpmbuild -bp kernel.spec
sed -i 's/CONFIG_IP_VS_TAB_BITS=.*$/CONFIG_IP_VS_TAB_BITS=20/g' /home/lvs/rpms/SOURCES/config-generic
cd /home/lvs/rpms/BUILD/kernel-2.6.32-431.el6/linux-2.6.32-431.el6.x86_64/
cd /usr/local/src/lvs/
tar xf Lvs-fullnat-synproxy.tar.gz
cd /home/lvs/rpms/BUILD/kernel-2.6.32-431.el6/linux-2.6.32-431.el6.x86_64/
patch -p1 < /usr/local/src/lvs/lvs-fullnat-synproxy/toa-2.6.32-220.23.1.el6.patch
sed -i '/WIMAX/a obj\-\$$CONFIG\_TOA$ \+\= toa/' net/Makefile
make -j8
make modules_install
make install
sed -i 's/default.*$/default=0/' /boot/grub/grub.conf
modprobe toa
# reload tao module
echo 'modprobe toa' >> /etc/rc.local

在Director上面安装并配置keepalived：

cd /usr/local/src/lvs/

tar xf Lvs-fullnat-synproxy.tar.gz

cd /usr/local/src/lvs/lvs-fullnat-synproxy

cd /home/lvs

cp / usr/local/src/lvs/lvs-fullnat-synproxy/lvs-tools.tar.gz ./
tar xf lvs-tools.tar.gz;
cd tools;

cd keepalived;
./configure --with-kernel-dir="/lib/modules/`uname -r`/build";
make;
make install;

mkdir /etc/keepalived/keepalived.d -pv

cp -a bin/keepalived /sbin/

cp -a keepalived/etc/init.d/keepalived.init /etc/init.d/keepalived

cp -a keepalived/etc/keepalived/keepalived.conf /etc/keepalived/keepalived.d

cp -a keepalived/etc/init.d/keepalived.sysconfig /etc/sysconfig/keepalived

安装ipvsadm：

cd tools/ipvsadm;
make;
make install;

keepalived配置文件如下：

MASTER:在lvs_dr01上面

[root@lvs_dr01 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
notification_email {
root@localhost
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id lvs_dr01
vrrp_mcast_group4 224.30.0.2
}

local_address_group laddr_g1 {
10.129.45.100
}

virtual_server_group vip {
10.129.45.254 80
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 53
priority 100
advert_int 1
nopreempt FALSE
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.129.45.254
}
}

virtual_server 10.129.45.254 80 {
delay_loop 6
lb_algo rr
lb_kind FNAT
protocol TCP
syn_proxy
laddr_group_name laddr_g1

real_server 10.129.45.102 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connet_port 80 #这里的port 80就是rs的port
}
}

real_server 10.129.45.103 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connet_port 80
}
}
real_server 10.129.45.104 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connet_port 80
}
}
real_server 10.129.45.105 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connet_port 80
}
}

}

BACKUP:

[root@lvs_dr02 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
notification_email {
root@localhost
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id lvs_dr02
vrrp_mcast_group4 224.30.0.2
}

local_address_group laddr_g1 {
10.129.45.106
}

virtual_server_group vip {
10.129.45.254 80
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 53
priority 50
advert_int 1
nopreempt FALSE
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.129.45.254
}
}

virtual_server 10.129.45.254 80 {
delay_loop 6
lb_algo rr
lb_kind FNAT
protocol TCP
syn_proxy
laddr_group_name laddr_g1

real_server 10.129.45.102 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connet_port 80
}
}

real_server 10.129.45.103 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connet_port 80
}
}
real_server 10.129.45.104 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connet_port 80
}
}
real_server 10.129.45.105 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connet_port 80
}
}

}

验证realserver是否成功加入集群;

验证toa模块功能：

在主机10.129.36.242上用浏览器访问 http://10.129.45.254/test1.html

在lvs_rs04上面查看日志，clientip是否为真实ip：下面为真实ip，说明成功，如果不成功应该是代理的Director 10.129.45.100的地址

参考博客：http://shanks.blog.51cto.com/3899909/1536539

lvs网卡调优： http://navyaijm.blog.51cto.com/4647068/1334671

<1>：millmon表示链路监测时间间隔，单位为ms，millmon=100表示每100ms监测一次链路连接状态，如果有一条不通，就转入另一条。这个值建议为100, 设成其它值可能导致不稳定
<2>：mode表示两张网卡的运行方式，0 表示load blance，1 表示热备（建议使用热备）

报错参考：http://www.iyunv.com/thread-66463-1-1.html

AUTHOR:网名为什么那么长

0 0