LB Cluster之二：LVS安装配置及实例

一、LVS安装、配置
1、查看内核是否支持ipvs

[root@localhost ~]# grep -i 'ipvs' /boot/config-2.6.32-573.el6.x86_64
# IPVS transport protocol load balancing support
# IPVS scheduler
# IPVS application helper
[root@localhost ~]# grep -i 'ipvs' -A 10 /boot/config-2.6.32-573.el6.x86_64
# IPVS transport protocol load balancing support
#
CONFIG_IP_VS_PROTO_TCP=y
CONFIG_IP_VS_PROTO_UDP=y
CONFIG_IP_VS_PROTO_AH_ESP=y
CONFIG_IP_VS_PROTO_ESP=y
CONFIG_IP_VS_PROTO_AH=y
CONFIG_IP_VS_PROTO_SCTP=y

#
# IPVS scheduler
#
CONFIG_IP_VS_RR=m
CONFIG_IP_VS_WRR=m
CONFIG_IP_VS_LC=m
CONFIG_IP_VS_WLC=m
CONFIG_IP_VS_LBLC=m
CONFIG_IP_VS_LBLCR=m
CONFIG_IP_VS_DH=m
CONFIG_IP_VS_SH=m
CONFIG_IP_VS_SED=m
--
# IPVS application helper
#
CONFIG_IP_VS_FTP=m
CONFIG_IP_VS_PE_SIP=m

#
# IP: Netfilter Configuration
#
CONFIG_NF_DEFRAG_IPV4=m
CONFIG_NF_CONNTRACK_IPV4=m
# CONFIG_NF_CONNTRACK_PROC_COMPAT is not set

[root@localhost ~]# uname -a
Linux localhost.localdomain 2.6.32-573.el6.x86_64 #1 SMP Thu Jul 23 15:44:03 UTC 2015 x86_64 x86_64 x86_64 GNU/Linux

注意：2.4.26，2.6.4及以后的kernel版本内核已经默认支持IPVS

2、安装ipvsadm

[root@localhost ~]# yum install -y ipvsadm
[root@localhost ~]# rpm -ql ipvsadm
/etc/rc.d/init.d/ipvsadm
/etc/sysconfig/ipvsadm-config
/sbin/ipvsadm
/sbin/ipvsadm-restore
/sbin/ipvsadm-save
/usr/share/doc/ipvsadm-1.26
/usr/share/doc/ipvsadm-1.26/README
/usr/share/man/man8/ipvsadm-restore.8.gz
/usr/share/man/man8/ipvsadm-save.8.gz
/usr/share/man/man8/ipvsadm.8.gz

3、ipvsadm命令的用法
管理集群服务：创建、修改、删除

管理集群服务的RS:添加、修改、移除
查看：统计数据、速率

1）管理集群服务
创建或修改：ipvsadm -A|E -t|u|f service-address [-s scheduler]
-A：添加
-E：修改
-t: 承载的应用层协议为基于TCP协议提供服务的协议；其server-address的格式为“VIP:PORT”例如：“192.168.100.30：80”
-f: 承载的应用层协议为基于TCP或UDP协议提供服务的协议,但此类报文经过iptables/netfilter打标记，即防火墙标记：其server-address的格式为“FWM”；例如：“10”
-s: scheduler 指明调度算法；默认为WLC

[root@localhost ~]# ipvsadm -A -t 172.16.100.30:80
[root@localhost ~]# ipvsadm -l
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  172.16.100.30:http wlc
[root@localhost ~]#

删除：ipvsadm -D -t|u|f service-address
查看：ipvsadm -l|L

[root@localhost ~]# ipvsadm -D -t  172.16.100.30:80
[root@localhost ~]# ipvsadm -l
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
[root@localhost ~]#

2）管理集群上的RS
添加或修改：ipvsadm -a|e -t|u|f service-address -r server-address [-g|i|m] [-w weight]
-r server-address: 指明RS，server-address格式一般为“IP[:PORT]”;注意：只有支持端口映射的LVS类型才应该此处显式定义端口；例如：-r 192.168.100.10:8080
[-g|i|m]: 指明lvs类型；省略时默认为dr类型
-g: gateway，意为dr类型
-i：ipip，意为tun类型
-m: masquerade,意为nat类型
[-w weight]：当前RS的权重
注意：仅对于支持加权调度的scheduler才有意义

[root@localhost ~]# ipvsadm -a -t 192.168.100.30:80 -r 172.16.100.10 -m -w 2
[root@localhost ~]# ipvsadm -a -t 192.168.100.30:80 -r 192.168.100.20 -m -w 5
[root@localhost ~]# ipvsadm -l
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.100.30:http wlc
-> 172.16.100.10:http           Masq    2      0          0
-> 192.168.100.10:http          Masq    2      0          0
-> 192.168.100.20:http          Masq    5      0          0
[root@localhost ~]#

删除：ipvsadm -d -t|u|f service-address -r server-address
清空所有集群服务的定义：ipvsadm -C
保存集群服务及RS的定义：
ipvsadm -S > /etc/sysconfig/ipvsadm
ipvsadm-save > /etc/sysconfig/ipvsadm
service ipvsadm save

[root@localhost ~]# cat /etc/sysconfig/ipvsadm
cat: /etc/sysconfig/ipvsadm: No such file or directory
[root@localhost ~]# ipvsadm -S
-A -t 192.168.100.30:http -s wlc
-a -t 192.168.100.30:http -r 172.16.100.10:http -m -w 2
-a -t 192.168.100.30:http -r 192.168.100.10:http -m -w 2
-a -t 192.168.100.30:http -r 192.168.100.20:http -m -w 5
[root@localhost ~]# cat /etc/sysconfig/ipvsadm
cat: /etc/sysconfig/ipvsadm: No such file or directory
[root@localhost ~]# service ipvsadm save
ipvsadm: Saving IPVS table to /etc/sysconfig/ipvsadm:      [  OK  ]
[root@localhost ~]# cat /etc/sysconfig/ipvsadm
-A -t 192.168.100.30:80 -s wlc
-a -t 192.168.100.30:80 -r 172.16.100.10:80 -m -w 2
-a -t 192.168.100.30:80 -r 192.168.100.10:80 -m -w 2
-a -t 192.168.100.30:80 -r 192.168.100.20:80 -m -w 5
[root@localhost ~]#

恢复集群服务及RS的定义：
ipvsadm -R < /etc/sysconfig/ipvsadm
ipvsadm-restore < /etc/sysconfig/ipvsadm
service ipvsadm restart

[root@localhost ~]# ipvsadm -C
[root@localhost ~]# ipvsadm -l
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
[root@localhost ~]# cat /etc/sysconfig/ipvsadm
-A -t 192.168.100.30:80 -s wlc
-a -t 192.168.100.30:80 -r 172.16.100.10:80 -m -w 2
-a -t 192.168.100.30:80 -r 192.168.100.10:80 -m -w 2
-a -t 192.168.100.30:80 -r 192.168.100.20:80 -m -w 5
[root@localhost ~]# ipvsadm -R < /etc/sysconfig/ipvsadm
[root@localhost ~]# ipvsadm -l
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.100.30:http wlc
-> 172.16.100.10:http           Masq    2      0          0
-> 192.168.100.10:http          Masq    2      0          0
-> 192.168.100.20:http          Masq    5      0          0
[root@localhost ~]#

3）查看规则
ipvsadm -l|L [options]
-c: 列出当前所有connection
--stats: 列出统计数据
--rates: 列出速率
-n|--numeric: 数字格式显示IP及端口，不作反解
--exact:精确值

[root@localhost ~]# ipvsadm -l -c
IPVS connection entries
pro expire state       source             virtual            destination
[root@localhost ~]# curl http://192.168.100.30 curl: (7) couldn't connect to host
[root@localhost ~]# curl http://192.168.100.30 curl: (7) couldn't connect to host
[root@localhost ~]# ipvsadm -l -c
IPVS connection entries
pro expire state       source             virtual            destination
TCP 00:08  CLOSE       192.168.100.30:50227 192.168.100.30:http 192.168.100.10:http
TCP 00:07  CLOSE       192.168.100.30:50226 192.168.100.30:http 192.168.100.20:http

[root@localhost ~]# ipvsadm -l --stats
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes
-> RemoteAddress:Port
TCP  192.168.100.30:http                 3        3        3      180      120
-> 172.16.100.10:http                  0        0        0        0        0
-> 192.168.100.10:http                 1        1        1       60       40
-> 192.168.100.20:http                 2        2        2      120       80

[root@localhost ~]# ipvsadm -l --rate
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port                 CPS    InPPS   OutPPS    InBPS   OutBPS
-> RemoteAddress:Port
TCP  192.168.100.30:http                 0        0        0        0        0
-> 172.16.100.10:http                  0        0        0        0        0
-> 192.168.100.10:http                 0        0        0        0        0
-> 192.168.100.20:http                 0        0        0        0        0

4）清空计数器
ipvsadm -Z [-t|u|f service-address]

二、实战案例
LVS-nat类型Director实现httpd集群负载均衡
1）实验环境：
OS:CentOS6.7
CIP:192.168.100.8 （windows）
VIP:192.168.100.30 （Director eth0）

DIP:192.168.200.30 (Director eth1)
R1IP:192.168.200.10 (gw 192.168.200.30)
R2IP:192.168.200.20 (gw 192.168.200.30)

2）配置Director

[root@localhost ~]# ipvsadm -A -t 192.168.100.30:80
[root@localhost ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.100.30:80 wlc
[root@localhost ~]#
[root@localhost ~]# ipvsadm -a -t 192.168.200.30:80 -r 192.168.200.20 -m -w 2
[root@localhost ~]# ipvsadm -a -t 192.168.200.30:80 -r 192.168.200.10 -m -w 1
[root@localhost ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.100.30:80 wlc
-> 192.168.200.10:80            Masq    1      0          0
-> 192.168.200.20:80            Masq    2      0          0
[root@localhost ~]# ipvsadm save
Try `ipvsadm -h' or 'ipvsadm --help' for more information.
[root@localhost ~]# service ipvsadm save
ipvsadm: Saving IPVS table to /etc/sysconfig/ipvsadm:      [  OK  ]
[root@localhost ~]# cat /etc/sysconfig/ipvsadm
-A -t 192.168.100.30:80 -s wlc
-a -t 192.168.100.30:80 -r 192.168.200.10:80 -m -w 1
-a -t 192.168.100.30:80 -r 192.168.200.20:80 -m -w 2

3）打开NAT转发功能

[root@localhost ~]# cat /proc/sys/net/ipv4/ip_forward
0
[root@localhost ~]# echo 1 > /proc/sys/net/ipv4/ip_forward  #临时更改
[root@localhost ~]# sed -i 's/net.ipv4.ip_forward =0/net.ipv4.ip_forward =1/' /etc/sysctl.conf      #永久更改
[root@localhost ~]# sysctl -p
net.ipv4.ip_forward = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296

4）测试

[root@localhost ~]# ab -n 10000 -c 1000 http://192.168.100.30/index.html [root@localhost ~]# ipvsadm -ln --stats
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes
-> RemoteAddress:Port
TCP  192.168.100.30:80               99630   630560   500972 45219839 55454909
-> 192.168.200.10:80               52544   315921   262136 21901574 29182095
-> 192.168.200.20:80               47086   314639   238836 23318265 26272814

[root@localhost ~]# ipvsadm -E -t 192.168.100.30 -s wrr
Zero port specified for non-persistent service
[root@localhost ~]# ipvsadm -E -t 192.168.100.30:80 -s wrr
[root@localhost ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.100.30:80 wrr
-> 192.168.200.10:80            Masq    1      0          0
-> 192.168.200.20:80            Masq    2      0          0
[root@localhost ~]# ipvsadm -Z
[root@localhost ~]# ipvsadm -ln --stats
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes
-> RemoteAddress:Port
TCP  192.168.100.30:80                   0        0        0        0        0
-> 192.168.200.10:80                   0        0        0        0        0
-> 192.168.200.20:80                   0        0        0        0        0
[root@localhost ~]# ab -n 50000 -c 1500 http://192.168.100.30/index.html This is ApacheBench, Version 2.3 <$Revision: 655654 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ Licensed to The Apache Software Foundation, http://www.apache.org/ 
Benchmarking 192.168.100.30 (be patient)
socket: Too many open files (24)
[root@localhost ~]# ab -n 50000 -c 1000 http://192.168.100.30/index.html [root@localhost ~]# ipvsadm -ln --stats
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes
-> RemoteAddress:Port
TCP  192.168.100.30:80               52728   340723   261325 24910676 28530623
-> 192.168.200.10:80               17576   105021    85682  7356740  9383766
-> 192.168.200.20:80               35152   235702   175643 17553936 19146857

抓包工具：

tcpdump -i eth0 -nn [src|dst] host IP and [src|dst] tcp|dcp 80

[root@localhost ~]# tcpdump -i eth0 -nn host 192.168.200.10 and tcp port 80
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 65535 bytes
21:40:57.293619 IP 192.168.100.99.55270 > 192.168.200.10.80: Flags [S], seq 2884297711, win 32768, options [mss 1460,nop,wscale 2,nop,nop,sackOK], length 0
21:40:57.293801 IP 192.168.200.10.80 > 192.168.100.99.55270: Flags [S.], seq 1386513891, ack 2884297712, win 14600, options [mss 1460,nop,nop,sackOK,nop,wscale 6], length 0
21:40:57.296749 IP 192.168.100.99.55270 > 192.168.200.10.80: Flags [.], ack 1, win 8192, length 0

LVS-dr类型Director实现httpd集群负载均衡1）实验环境：OS:CentOS6.7CIP:192.168.200.8 （windows）VIP:192.168.200.90 （Director eth0）
DIP:192.168.200.30 (Director eth0:0)R1IP:192.168.200.10 (lo:0 192.168.200.90 broadcast 192.168.200.90 netmask 255.255.255.255)R2IP:192.168.200.20 (lo:0 192.168.200.90 broadcast 192.168.200.90 netmask 255.255.255.255)
2）配置R1,R2不响应VIP的ARP请求及接收、从lo接口响应VIP

[root@localhost conf]# cat  /proc/sys/net/ipv4/conf/all/arp_announce
0
[root@localhost conf]# cat >> /etc/sysctl.conf <<EOF
>net.ipv4.conf.all.arp_ignore = 1
>net.ipv4.conf.lo.arp_ignore = 1
>net.ipv4.conf.all.arp_announce = 2
>net.ipv4.conf.lo.arp_announce = 2
> EOF
[root@xxj ~]# sysctl -p
net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
[root@xxj ~]# cat /proc/sys/net/ipv4/conf/lo/arp_announce
2
[root@localhost conf]# cat /proc/sys/net/ipv4/conf/lo/arp_ignore
1

[root@xxj ~]# route add -host 192.168.200.90 dev lo:0
[root@xxj ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
192.168.200.90  0.0.0.0         255.255.255.255 UH    0      0        0 lo
0.0.0.0         192.168.100.1   0.0.0.0         UG    0      0        0 eth0

配置过程总结：
Director：
(1) VIP配置在物理接口的别名上
ifconfig INTERFACE:ALIAS $vip broadcast $vip netmask 255.255.255.255
(2) 配置路由信息

route add -host $vip dev INTEFACE:ALIAS

RS:
(1) 先修改内核参数
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
(2) VIP配置在lo的别名上

ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up (3) 配置路由信息
route add -host $vip dev lo:0
（4）配置ipvsadm

（5）测试

DR类型director脚本示例：

#!/bin/bash
#
vip=172.16.100.7
rip=('172.16.100.8' '172.16.100.9')
weight=('1' '2')
port=80
scheduler=rr
ipvstype='-g'

case $1 in
start)
iptables -F -t filter
ipvsadm -C

ifconfig eth0:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip dev eth0:0
echo 1 > /proc/sys/net/ipv4/ip_forward      # 脚本中为什么还用临时生效的方法更改，有时间再修改了

ipvsadm -A -t $vip:$port -s $scheduler
[ $? -eq 0 ] && echo "ipvs service $vip:$port added."  || exit 2
for i in `seq 0 $[${#rip[@]}-1]`; do
ipvsadm -a -t $vip:$port -r ${rip[$i]} $ipvstype -w ${weight[$i]}
[ $? -eq 0 ] && echo "RS ${rip[$i]} added."
done
touch /var/lock/subsys/ipvs
;;
stop)
echo 0 > /proc/sys/net/ipv4/ip_forward
ipvsadm -C
ifconfig eth0:0 down
rm -f /var/lock/subsys/ipvs
echo "ipvs stopped."
;;
status)
if [ -f /var/lock/subsys/ipvs ]; then
echo "ipvs is running."
ipvsadm -L -n
else
echo "ipvs is stopped."
fi
;;
*)
echo "Usage: `basename $0` {start|stop|status}"
exit 3
;;
esac

DR类型RS脚本示例：

#!/bin/bash
#
vip=172.16.100.7
interface="lo:0"

case $1 in
start)
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce

ifconfig $interface $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip dev $interface
;;
stop)
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce

ifconfig $interface down
;;
status)
if ifconfig lo:0 |grep $vip &> /dev/null; then
echo "ipvs is running."
else
echo "ipvs is stopped."
fi
;;
*)
echo "Usage: `basename $0` {start|stop|status}"
exit 1
esa

本文出自 “xiexiaojun” 博客，请务必保留此出处http://xiexiaojun.blog.51cto.com/2305291/1703611