keepalived+lvs/nginx 实现调度器高可用

友情提醒：本文实验环境 centos 6.6 X86_64 + vmware 10，文中命令请谨慎使用
一 关于keepalived的理论
待补
二 实验拓扑和实验环境设定：

主机	主机名和IP	角色
Test06	Test06.lijun.com eth2:172.16.100.6/24	后台web服务器，提供HTTPD服务
Test07	Test07.lijun.com eth2:172.16.100.7/24	后台web服务器，提供HTTPD服务
Test03	Test03.lijun.com eth1:192.168.100.3/24 eth2:172.16.100.3/24	前台调度器
Test04	Test04.lijun.com eth1:192.168.100.4/24 eth2:172.16.100.4/24	前台调度器
client	192.168.100.100/24	测试机
IP：192.168.100.10/24 虚拟的后台web资源IP，是client访问的唯一地址 IP:172.16.100.10/24 下文lvs高可用时虚拟的DIP地址

实验拓扑：




三 keepalived实现LVS调度器高可用

*lvs使用NET网络模型
1）后台web服务器设定：
Test07上：

#关闭iptables和selinux防止干扰实验
[root@Test07 ~]#serivce iptables stop
[root@Test07 ~]#setenforce 0
#设定ip
[root@Test07 ~]#ip link set up dev eth2
[root@Test07 ~]#ip addr 172.16.100.7/24 dev eth2
#因做的lvs的nat模型，故设定该路由
[root@Test07 ~]#ip route add default via  172.16.100.10
#安装httpd软件，并设定主页内容
[root@Test07 ~]#yum -y install httpd
[root@Test07 ~]#echo "<h1>Test07,ip address is 100.7</h1>">/var/www/html/index.html
#启动httpd服务
[root@Test07 ~]#service httpd start

Test06 上：

#同上不解释
[root@Test06 ~]#serivce iptables stop
[root@Test06 ~]#setenforce 0
[root@Test06 ~]#ip link set up dev eth2
[root@Test06 ~]#ip addr 172.16.100.6/24 dev eth2
[root@Test06 ~]#ip route add default via  172.16.100.10
[root@Test06 ~]#yum -y install httpd
[root@Test06 ~]#echo "<h1>This is Test06,my ip address is 172.16.100.6</h1>">/var/www/html/index.html
[root@Test06 ~]#service httpd start

2）Test03调度器环境的设定：

#关闭iptables和selinux放置干扰实验，另做为lvs调度器必须清空input链规则
[root@Test03 ~]#service iptables stop
[root@Test03 ~]#setenforce 0
#因为做lvs nat模型调度器故设定IPv4的数据包转发
[root@Test03 ~]#echo 1>/proc/sys/net/ipv4/ip_forward
#设定IP地址
[root@Test03 ~]#ip addr add 172.16.100.3/24 dev eth2
[root@Test03 ~]#ip addr add 192.168.100.3/24 dev  eth1
#增加kpadmin用户，用来接受邮件使用
[root@Test03 ~]#useradd kpadmin
[root@Test03 ~]#echo 'redhat' | passwd --stdin kpadmin

测试同后台web服务器的连通性：



3）Test03上keepalived的设定：

#从centos6.4开始keepalive就成为系统安装树的成员，这样使用yum直接安装
[root@Test03 ~]#yum -y install keepalived
[root@Test03 ~]# cd /etc/keepalived/
#备份配置文件，这是一个好习惯
[root@Test03 keepalived]# cp keepalived.conf{,.bak}

[root@Test03 keepalived]#vim keepalived.conf
! Configuration File for keepalived
#全局设定，关于警示邮件的发送设定
global_defs {
notification_email {
kpadmin@127.0.0.1
}
notification_email_from kaadmin@lijun.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVSFOR80
}
#定义对lvs调度器本身的检查方式
vrrp_script chk_mt_down {
script "[[ -f /var/lock/subsys/lvsdown ]] && exit 1 || exit 0"
interval 1
weight -5
}
#定义vrrp虚拟资源组，很明显这台机器做主节点
vrrp_instance VI_1 {
state MASTER
interface eth1
virtual_router_id 57
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass VI1pass
}
#因为是lvs nat模型，故这里的资源IP设定2个一个是vip一个是dip
virtual_ipaddress {
192.168.100.10/24 dev eth1 label eth1:0
172.16.100.10/24 dev  eth2 label eth2:0
}
track_script {
chk_mt_down
}
}
#这里定义lvs的集群
virtual_server 192.168.100.10 80 {
delay_loop 6
lb_algo rr
lb_kind NAT
nat_mask 255.255.255.0
protocol TCP
real_server 172.16.100.6 80 {
weight 1
#使用HTTP_GET方式检查后台服务器的存活
HTTP_GET {
url {
path /index.html
status_code 200
}
connect_timeout 2
nb_get_retry 3
delay_before_retry 1
}
}
real_server 172.16.100.7 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
connect_timeout 2
nb_get_retry 3
delay_before_retry 1
}
}
}
[root@Test03 keepalived]#service keepalived  start

观察资源Ip的设定：




4）Test04调度器上环境设定：

#同上2）不解释
[root@Test04 ~]#service iptables stop
[root@Test04 ~]#setenforce 0
[root@Test04 ~]#echo 1>/proc/sys/net/ipv4/ip_forward
[root@Test04 ~]#ip addr add 172.16.100.4/24 dev eth2
[root@Test04 ~]#ip addr add 192.168.100.4/24 dev  eth1
[root@Test04 ~]#useradd kpadmin
[root@Test04 ~]#echo 'redhat' | passwd --stdin kpadmin

5）Test04上keepalived的设定：

[root@Test04 ~]#yum -y install keepalived
#为保证配置文件中特殊部分的设定，这里直接copyTest03的配置，并进行更改
[root@Test04 ~]#scp 192.168.100.3:/etc/keepalived/keepalived.conf  /etc/keepalived/

[root@Test04 ~]#vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
kpadmin@127.0.0.1
}
notification_email_from kaadmin@lijun.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVSFOR80
}
vrrp_script chk_mt_down {
script "[[ -f /var/lock/subsys/lvsdown ]] && exit 1 || exit 0"
interval 1
weight -5
}
#Test03是主节点，这台Test04做辅助节点使用
vrrp_instance VI_1 {
state BACKUP
interface eth1
virtual_router_id 57
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass VI1pass
}
virtual_ipaddress {
192.168.100.10/24 dev eth1 label eth1:0
172.16.100.10/24 dev  eth2 label eth2:0
}
track_script {
chk_mt_down
}
}
virtual_server 192.168.100.10 80 {
delay_loop 6
lb_algo rr
lb_kind NAT
nat_mask 255.255.255.0
protocol TCP
real_server 172.16.100.6 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
connect_timeout 2
nb_get_retry 3
delay_before_retry 1
}
}
real_server 172.16.100.7 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
connect_timeout 2
nb_get_retry 3
delay_before_retry 1
}
}
}
[root@Test04 ~]#service keepalived start

6）客户端访问观察：






7）在主节点Test03上建立lvsdown文件观察资源IP的转移情况







8）客户端访问测试：








9）在主节点Test03上删除lvsdown文件，观察资源IP的是否会转移：




10）将后台web服务停止一台，看下客户端通过lvs能访问什么呢：




四 keepalived 实现nginx代理调度器的高可用
*这里nginx只实现简单的代理功能
实验环境接上文
11）设定nginx 的代理功能：
nginx的安装这里忽略，请自行准备，这里给出nginx的配置文件，莫喷我，懒!!!
Test03,Test04上均安装nginx，均使用下面的配置文件

# grep -E -v '(^[[:space:]]{0,}#|^$)' /usr/local/nginx/conf/nginx.conf
worker_processes  1;
events {
worker_connections  1024;
}
http {
upstream backwebserver {
server 172.16.100.6 weight=1;
server 172.16.100.7 weight=1;
}
include       mime.types;
default_type  application/octet-stream;
sendfile        on;
keepalive_timeout  65;
server {
listen       80;
server_name  localhost;
location / {
proxy_pass http://backwebserver/; }
error_page   500 502 503 504  /50x.html;
location = /50x.html {
root   html;
}
}
}
#/usr/local/nginx/sbin/nginx

12）Test03上的设定keepalived

#恢复机器环境
[root@Test03 ~]#echo 0 > /proc/sys/net/ipv4/ip_forward
[root@Test03 ~]#ifconfig down eth1
[root@Test03 ~]#ifconfig eth1 192.168.100.3 netmask 255.255.255.0 up
[root@Test03 ~]#ifconfig down eth2
[root@Test03 ~]#ifconfig eth2 172.16.100.3 netmask 255.255.255.0 up
[root@Test03 ~]#service keepalived stop
[root@Test03 ~]# ipvsadm -C

#恢复keepalive的主机环境
[root@Test03 ~]#cd  /etc/keepalived/
[root@Test03 keepalived]#rm -rf keepalived.conf
[root@Test03 keepalived]#cp keepalived.conf.bak keepalived.conf

#从新定义keepalived

[root@Test03 keepalived]# vim keepalived.conf
! Configuration File for keepalived

global_defs {
notification_email {
kpadmin@localhost
}
notification_email_from kaadmin@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVSFOR80
}
#这是定义对nginx的检测，并做为资源IP是否转移的依据
vrrp_script chk_nginx {
script "killall -0 nginx &> /dev/null"
interval 1
weight -5
}
vrrp_instance no1 {
state MASTER
interface eth1
virtual_router_id 57
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass VI1pass
}
virtual_ipaddress {
192.168.100.10/24 dev eth1 label eth1:0
}
track_script {
chk_nginx
}
#这里定义了2个命令，根据nginx的检查结果来执行，使用的脚本见下文
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
}
#定义脚本，实现当nginx状态改变后，发送邮件通知
[root@Test03 keepalived]#touch notify.sh && chmod +x notify.sh

[root@Test03 keepalived]# vim notify.sh
#!/bin/bash
#The scripts userd for send mail when nginx change the state
vip=192.168.100.10
contact='kpadmin@localhost'
notify() {
mailsubject="`hostname` to be $1: $vip floating"
mailbody="`date '+%F %H:%M:%S'`: vrrp transition, `hostname` changed to be $1"
echo $mailbody | mail -s "$mailsubject" $contact
}

case "$1" in
master)
notify master
exit 0
;;
backup)
notify backup
exit 0
;;
*)
echo 'Usage: `basename $0` {master|backup}'
exit 1
;;
esac

[root@Test03 keepalived]# service keepalived start

13)Test04上的设定
#恢复环境设定

[root@Test04 ~]#echo 0 > /proc/sys/net/ipv4/ip_forward
[root@Test04 ~]#ifconfig down eth1
[root@Test04 ~]#ifconfig eth1 192.168.100.4 netmask 255.255.255.0 up
[root@Test04 ~]#ifconfig down eth2
[root@Test04 ~]#ifconfig eth2 172.16.100.4 netmask 255.255.255.0 up
[root@Test04 ~]#service keepalived stop
[root@Test04 ~]# ipvsadm -C
[root@Test04 ~]#cd  /etc/keepalived/
[root@Test04 keepalived]#rm -rf keepalived.conf
[root@Test04 keepalived]#cp keepalived.conf.bak keepalived.conf

#从新定义keepalived

[root@Test04 keepalived]# vim keepalived.conf
! Configuration File for keepalived

global_defs {
notification_email {
kpadmin@localhost
}
notification_email_from kaadmin@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVSFOR80
}

vrrp_script chk_nginx {
script "killall -0 nginx &> /dev/null"
interval 1
weight -5
}
vrrp_instance no1 {
state BACKUP
interface eth1
virtual_router_id 57
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass VI1pass
}
virtual_ipaddress {
192.168.100.10/24 dev eth1 label eth1:0
}
track_script {
chk_nginx
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
}

[root@Test04 keepalived]# vim notify.sh
#!/bin/bash
#The scripts userd for send mail when nginx change the state
vip=192.168.100.10
contact='kpadmin@localhost'
notify() {
mailsubject="`hostname` to be $1: $vip floating"
mailbody="`date '+%F %H:%M:%S'`: vrrp transition, `hostname` changed to be $1"
echo $mailbody | mail -s "$mailsubject" $contact
}

case "$1" in
master)
notify master
exit 0
;;
backup)
notify backup
exit 0
;;
*)
echo 'Usage: `basename $0` {master|backup}'
exit 1
;;
esac
[root@Test04 keepalived]# service keepalived start

14）客户端测试：









15）停止主节点上nginx服务，观察资源IP的转移：







16）观察是否有邮件提醒：



17）启动Test03上的nginx看资源IP的情况




这两天在搞python的面向对象的编程，文章写的有点糙，见谅！！