Linux系统架构-（LB-HA集群）之LVS配置

LB集群之LVS
ha lb load balance
负载均衡软件 nginx、lvs、keepalived
设备F5、netscale

lvs有三种模式
1.NAT
2.TUN
3.DR

调度算法:rr、wrr、dh、sh
动态算法：wlc、lc、lblc、lblcr

LVS的NAT模式配置：
准备三台设备。1台为转发，其他2台为提供的服务。
为区分开:
1号机为dir,
2号机为rs1,
3号机为rs2

1号机：
[root@wangchao ~]# hostname dir
[root@wangchao ~]# ifconfig
eth0 inet addr:192.168.137.22
eth1 inet addr:192.168.2.22
//1号机准备两个网卡，假设eth0连接的是内网，为内网地址；eth1连接的是外网，为外网地址

2号机
[root@master ~]# hostname rs1
[root@master ~]# bash
[root@rs1 ~]# ifconfig
eth0 inet addr:192.168.137.21

3号机
[root@slave ~]# hostname rs2
[root@slave ~]# bash
[root@rs2 ~]# ifconfig
eth0 inet addr:192.168.137.23

dir上：
root@wangchao ~]# bash
[root@dir ~]# yum install -y ipvsadm
[root@dir ~]# vim/usr/local/sbin/lvs_nat.sh //为方便管理写一脚本，内容如下：
#! /bin/bash
# director 服务器上开启路由转发功能:
echo 1 > /proc/sys/net/ipv4/ip_forward
# 关闭icmp的重定向
echo 0 >/proc/sys/net/ipv4/conf/all/send_redirects
echo 0 >/proc/sys/net/ipv4/conf/default/send_redirects
echo 0 >/proc/sys/net/ipv4/conf/eth0/send_redirects
echo 0 >/proc/sys/net/ipv4/conf/eth1/send_redirects
# director 设置nat防火墙
iptables -t nat -F
iptables -t nat -X
iptables -t nat -A POSTROUTING -s192.168.137.0/24 -j MASQUERADE
# director设置ipvsadm
IPVSADM='/sbin/ipvsadm'
$IPVSADM -C
$IPVSADM -A -t 192.168.2.22:80 -s rr
$IPVSADM -a -t 192.168.2.22:80 -r192.168.137.21:80 -m -w 1
$IPVSADM -a -t 192.168.2.22:80 -r192.168.137.23:80 -m -w 1

[root@dir ~]# sh /usr/local/sbin/lvs_nat.sh
[root@dir ~]# ipvsadm -l
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.203.141.18:http lc persistent 300
-> 192.168.137.21:http Masq 1 0 0
-> 192.168.137.23:http Masq 1 0 0
[root@dir ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.2.22:80 rr
-> 192.168.137.21:80 Masq 1 0 1
-> 192.168.137.23:80 Masq 1 0 0
//配置成功

rs1上：
[root@rs1 ~]# vim/etc/sysconfig/network-scripts/ifcfg-eth0
GATEWAY=192.168.137.22 //网关需设置成dir的IP地址
[root@rs1 ~]# ifdown eth0; ifup eth0

rs2上：
[root@rs2 ~]# vim/etc/sysconfig/network-scripts/ifcfg-eth0
GATEWAY=192.168.137.22
[root@rs2 ~]# ifdown eth0; ifup eth0
[root@rs1 ~]# service NetworkManager stop
[root@rs1 ~]# chkconfig NetworkManager off
[root@rs1 ~]# service network restart

rs1上：
[root@rs1 ~]# /etc/init.d/nginx start
Starting nginx: [ OK ]
[root@rs1 ~]# netstat -lnp |grep nginx
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 9423/nginx
[root@rs1 ~]# curl localhost
master

rs2上：
[root@rs2 ~]# netstat -lnp |grep nginx
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 4871/nginx
[root@rs2 ~]# curl localhost
slave

dir上查看外网地址：
[root@dir ~]# ifconfig
eth1 inet addr:192.168.2.22

window客户端多次访问192.168.2.22时,出现一次master，一次slave





[root@wangchao ~]# curl 192.168.2.22
master
[root@wangchao ~]# curl 192.168.2.22
slave
[root@wangchao ~]# curl 192.168.2.22
master
[root@wangchao ~]# curl 192.168.2.22
slave



[root@dir ~]# vim/usr/local/sbin/lvs_nat.sh
#! /bin/bash
# director 服务器上开启路由转发功能:
echo 1 > /proc/sys/net/ipv4/ip_forward
# 关闭icmp的重定向
echo 0 > /proc/sys/net/ipv4/conf/all/send_redirects
echo 0 >/proc/sys/net/ipv4/conf/default/send_redirects
echo 0 >/proc/sys/net/ipv4/conf/eth0/send_redirects
echo 0 >/proc/sys/net/ipv4/conf/eth1/send_redirects
# director 设置nat防火墙
iptables -t nat -F
iptables -t nat -X
iptables -t nat -A POSTROUTING -s192.168.137.0/24 -j MASQUERADE
# director设置ipvsadm
IPVSADM='/sbin/ipvsadm'
$IPVSADM -C
$IPVSADM -A -t 192.168.2.22:80 -s wrr
$IPVSADM -a -t 192.168.2.22:80 -r192.168.137.21:80 -m -w 2
$IPVSADM -a -t 192.168.2.22:80 -r192.168.137.23:80 -m -w 1

[root@dir ~]#sh /usr/local/sbin/lvs_nat.sh
多次访问192.168.2.22，出现两次master,一次slave
[root@dir ~]# curl 192.168.2.22
master
[root@dir ~]# curl 192.168.2.22
master
[root@dir ~]# curl 192.168.2.22
slave
[root@dir ~]# curl 192.168.2.22
master
[root@dir ~]# curl 192.168.2.22
master
[root@dir ~]# curl 192.168.2.22
slave
[root@dir ~]# curl 192.168.2.22
master
[root@dir ~]# curl 192.168.2.22
master
[root@dir ~]# curl 192.168.2.22
slave



LVS的DR设置
清空之前的规则
dir上：
[root@dir ~]# ipvsadm -ln //查看
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.2.22:80 wrr
-> 192.168.137.21:80 Masq 2 0 0
-> 192.168.137.23:80 Masq 1 0 0
[root@dir ~]# ipvsadm -C
[root@dir ~]# ipvsadm -ln
[root@dir ~]# iptables -t nat -F
[root@dir ~]# ifdown eth1

[root@dir ~]# vim /usr/local/sbin/lvs_dr.sh
#! /bin/bash
echo 1 > /proc/sys/net/ipv4/ip_forward
ipv=/sbin/ipvsadm
vip=192.168.137.100
rs1=192.168.137.21
rs2=192.168.137.23
ifconfig eth0:0 $vip broadcast $vip netmask255.255.255.255 up
route add -host $vip dev eth0:0
$ipv -C
$ipv -A -t $vip:80 -s wrr
$ipv -a -t $vip:80 -r $rs1:80 -g -w 1
$ipv -a -t $vip:80 -r $rs2:80 -g -w 1

[root@dir ~]# sh !$
sh /usr/local/sbin/lvs_dr.sh
[root@dir ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.137.100:80 rr
-> 192.168.137.21:80 Route 1 0 0
-> 192.168.137.23:80 Route 1 0 0

rs1上：
[root@rs1 ~]# vim/usr/local/sbin/lvs_dr_rs.sh
#! /bin/bash
vip=192.168.137.100
ifconfig lo:0 $vip broadcast $vip netmask255.255.255.255 up
route add -host $vip lo:0
echo"1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo"2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo"1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo"2" >/proc/sys/net/ipv4/conf/all/arp_announce

[root@rs1 ~]# sh/usr/local/sbin/lvs_dr_rs.sh
[root@rs1 ~]# vim/etc/sysconfig/network-scripts/ifcfg-eth0
GATEWAY=192.168.137.1
[root@rs1 ~]# service network restart

rs2上：
[root@rs2 ~]# vim/usr/local/sbin/lvs_dr_rs.sh
#! /bin/bash
vip=192.168.137.100
ifconfig lo:0 $vip broadcast $vip netmask255.255.255.255 up
route add -host $vip lo:0
echo"1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo"2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo"1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo"2" >/proc/sys/net/ipv4/conf/all/arp_announce

[root@rs2 ~]# sh !$
sh /usr/local/sbin/lvs_dr_rs.sh
[root@rs2 ~]# vim /etc/sysconfig/network-scripts/ifcfg-eth0
GATEWAY=192.168.137.1
[root@rs2 ~]# service network restart

客户端访问：



浏览器有些时候不准，我们用curl测试，打开第四台设备测试



一次master,一次slave

LVS结合keepalived配置
在以上配置中，如果有一台设备宕机了，还是执行rr,那么问题就来了

rs1上关闭服务(模拟宕机)
[root@rs1 ~]# /etc/init.d/nginx stop

客户端测试：
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
slave
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
curl: (7) Failed to connect to192.168.137.100 port 80: Connection refused
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
slave
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
curl: (7) Failed to connect to192.168.137.100 port 80: Connection refused
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
slave
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
curl: (7) Failed to connect to192.168.137.100 port 80: Connection refused
wangchao@wangchao-virtual-machine:~$



出现了访问一次成功，一次失败。因为转发到了服务停止的设备上

现可安装第三方软件解决这个问题keepalive(负载均衡与高可用与一体)
keepalive需设置主从
dir上：
[root@dir ~]# ipvsadm -C //清空之前的规则
[root@dir ~]# yum install -y keepalived

为节省一点资源，从就在rs2上做了
rs2上：
[root@rs2 ~]# yum install -y keepalived

dir上：
[root@dir ~]# vim/etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
state MASTER #备用服务器上为BACKUP
interface eth0
virtual_router_id 51
priority 100 #备用服务器上为90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.137.100
}
}
virtual_server 192.168.137.100 80 {
delay_loop 6 #(每隔10秒查询realserver状态)
lb_algo wlc #(lvs算法)
lb_kind DR #(Direct Route)
persistence_timeout 60 #(同一IP的连接60秒内被分配到同一台realserver)
protocol TCP #(用TCP协议检查realserver状态)

real_server 192.168.137.21 80 {
weight 100 #(权重)
TCP_CHECK {
connect_timeout 10 #(10秒无响应超时)
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.137.23 80 {
weight 100
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}

[root@dir~]# scp /etc/keepalived/keepalived.conf192.168.137.23:/etc/keepalived/keepalived.conf
//配置文件拷贝到从上

rs2上：
[root@rs2 ~]# vim/etc/keepalived/keepalived.conf
state BACKUP
priority 90

dir上：
[root@dir ~]# ipvsadm -ln //查看无规则

[root@dir ~]# ifconfig //存在虚拟IP
eth0:0 inet addr:192.168.137.100
[root@dir ~]# /etc/init.d/keepalived start

rs2上也启动
[root@rs2 ~]# /etc/init.d/keepalived start

dir上：
[root@dir ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.137.100:80 wlc persistent 60
-> 192.168.137.23:80 Route 100 0 0

rs1上：启动之前停掉的服务
[root@rs1 ~]# /etc/init.d/nginx start

dir上再查看
[root@dir ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.137.100:80 wlc persistent 60
-> 192.168.137.21:80 Route 100 0 0
-> 192.168.137.23:80 Route 100 0 0

//只有当设备活的时候才会去转发

[root@dir ~]# ip addr
inet 192.168.137.100/32

客户端可以访问：
现停止一台设备提供服务
rs1上停止服务
[root@rs1 ~]# curl localhost
master
[root@rs1 ~]# /etc/init.d/nginx stop

客户端测试：
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
master
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
curl: (7) Failed to connect to192.168.137.100 port 80: Connection refused
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
slave
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
slave
wangchao@wangchao-virtual-machine:~$ curl192.168.137.100
slave



//较短时间内自动切换，切换到正常提供服务的设备上
dir上查看规则
[root@dir ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.137.100:80 wlc persistent 60
-> 192.168.137.23:80 Route 100 0 0

//只剩下一个了
rs1上，再启动之
root@rs1 ~]# /etc/init.d/nginx start

dir上：
[root@dir ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.137.100:80 wlc persistent 60
-> 192.168.137.21:80 Route 100 0 0
-> 192.168.137.23:80 Route 100 0 0
//自动加回，变成两个了

[root@dir ~]# ip addr
eth0:
inet 192.168.137.22/24 brd 192.168.137.255scope global eth0
inet 192.168.137.100/32 brd 192.168.137.100scope global eth0:0
//加载了虚拟IP：192.168.137.100

[root@rs2 ~]# ip addr
eth0：
inet 192.168.137.23/24 brd 192.168.137.255scope global eth0
//rs2上为加载虚拟IP192.168.137.100

当主keepalived上stop后
[root@dir ~]# /etc/init.d/keepalived stop

从上
[root@rs2 ~]# ip addr
eth0:
inet 192.168.137.23/24 brd 192.168.137.255scope global eth0
inet 192.168.137.100/32 scope global eth0
//自动加载虚拟IP