Nginx配合keepalived实现LVS负载均衡--生产环境实际案例 推荐

前言：由于服务器资源有限，keepalived未配主主、只配了一个VIP。本人在香港机房生产环境部署的实际案例，从2013年9月开始运行，服务从未挂过。由于涉及到生产环境，系统展示就不贴出来了。希望大家给予意见！
Webserver部署的是lnmp环境，采用我自己写的一键编译安装脚本，安装目录在/data/webserver/。如大家想借鉴我的编译安装脚本，请留言向我索取。
一、架构规划
1、服务器IP地址规划VIP：192.168.1.6real_server1:192.168.1.7real_server2:192.168.1.8WebServer1:192.168.1.9WebServer2:192.168.1.10memcache: 192.168.1.11mysql主：192.168.1.12mysql从：192.168.1.22、服务器操作系统所使用的操作系统均为CentOS 6.4 3、网络拓扑图




二、负载服务器配置

1.LVS主服务器配置安装yum install keepalived ipvsadm
chkconfig keepalived onvim /etc/ keepalived/keepalived.conf

global_defs {
notification_email {
yunwei@admin.com
}
notification_email_from
smtp_server 127.0.0.1
#smtp_connect_timeout 30
router_id LVS_DEVEL_1
}
vrrp_instance VI_1 {
state MASTER
interface eth1
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.1.6
}
}
virtual_server 192.168.1.6 80 {
delay_loop 6
#lb_algo wrr
lb_algo rr
lb_kind DR
#persistence_timeout 300
protocol TCP
real_server 192.168.1.7 80 {
weight 3
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.1.8 80 {
weight 3
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}

启动keepalived进程 /etc/init.d/keepalived start
查看本机VIP
2.LVS前端服务器配置（real_server）
2.1 vim /usr/local/bin/lvs_real

VIP=192.168.1.6
case "$1" in
start)
echo " start LVS of REALServer"
/sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
;;
stop)
/sbin/ifconfig lo:0 down
echo "close LVS Directorserver"
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
;;
*)
echo "Usage: $0 {start|stop}"
exit 1
esac

启动lvs前端真实IP： lvs_real start
2.2 编译安装nginx，此处省略（使用我自己写的一键编译安装脚本安装）
2.3 LVS前端服务器nginx配置

server {
listen 80;
server_name sandy.com;
location / {
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://sandy/; }
location /coocaa_status {
stub_status on;
access_log /data/log/nginx_status.log;
auth_basic "NginxStatus";
allow 192.168.0.58;
deny all;
}
access_log /data/log/access.log access;
error_log /data/log/error.log warn;
}
upstream sandy {
sticky;
server 192.168.1.9:80;   #后端webserver1内网IP
server 192.168.1.10:80;  #后端webserver2内网IP
}
}

三、Webserver服务器配置
1.编译安装nginx、php，此处省略（使用我自己写的一键编译安装脚本安装）
2.webserver服务器nginx配置，如下

user nginx;
worker_processes 8;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
error_log /data/log/error.log debug;
pid /data/webserver/nginx/logs/nginx.pid;
worker_rlimit_nofile 65535;
events {
use epoll;
worker_connections 65535;
}
http {
include mime.types;
default_type application/octet-stream;
charset utf-8;
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 50m;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
server_tokens off;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 3000;
fastcgi_read_timeout 3000;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 128k;
gzip on;
gzip_min_length 1k;
gzip_buffers 4  16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
log_format access '\$remote_addr - \$remote_user [\$time_local] "\$request" ' '\$status \$body_bytes_sent "\$http_referer" '
'"\$http_user_agent" "\$http_x_forwarded_for" - "\$http_soapaction"';
server
{
listen 80;
server_name 0.0.0.0; #webserver外网ip
index index.html index.php;
root   /data/www/cloudservice/;
#autoindex on;
location = /favicon.ico {
log_not_found off;
access_log off;
}
location /Framework/tvos/ {
try_files \$uri \$uri/ /Framework/tvos/index.php?\$args;
}
# nginx status
location /coocaa_status {
stub_status on;
access_log /data/log/nginx_status.log;
auth_basic "NginxStatus";
allow 192.168.0.58;
deny all;
}
location ~ \.php$ {
root           /data/www/cloudservice;
fastcgi_pass   127.0.0.1:9000;
fastcgi_index  index.php;
fastcgi_param  SCRIPT_FILENAME  /data/www/cloudservice/$fastcgi_script_name;
include        fastcgi_params;
}
location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)\$
{
expires 30d;
}
location ~ .*\.(js|css)?\$
{
expires 1h;
}
access_log /data/log/access.log access;
}
}

3.PHP-fpm配置如下：

[global]
pid = /data/webserver/php/var/run/php-fpm.pid
error_log = /data/log/php-fpm.error.log
log_level = error
[www]
listen = 127.0.0.1:9000
user = nginx
group = nginx
pm = dynamic
;pm= static
pm.max_children = 200
pm.start_servers = 32
pm.min_spare_servers = 32
pm.max_spare_servers = 200
pm.max_requests = 10240
request_terminate_timeout = 900
php_admin_flag[log_errors] = on
php_value[session.save_handler] = files
php_value[session.save_path] = /tmp
catch_workers_output = yes
access.log = /data/log/php-fpm.access.log
request_slowlog_timeout = 5
slowlog = /data/log/www-slow.log
pm.status_path = /status
rlimit_files = 65535

四、Memcache服务器部署
1.安装memcached

yum -y install libevent libevent-devel make gcc gcc-c++ build-essential make

memcached (){
pwd
wget -c http://42.120.20.242/memcached-1.4.15.tar.gz tar zxvf memcached-1.4.15.tar.gz
cd memcached-1.4.15/
./configure --with-libevent=/usr/local --prefix=/usr/local/memcached
make &&make install
cd ../

2.开启memcached实例，本项目中开了两个实例。如下
memcached -d -m 2500 -u root -l 192.168.1.13 -p 11211 -c 20480 -P /tmp/memcached.pid –v
memcached -d -m 2500 -u root -l 192.168.1.13 -p 11212 -c 20480 -P /tmp/memcached2.pid –v

五、Mysql服务器配置
1.编译安装mysql5.5.29，此处省略（使用我自己写的一键编译安装脚本安装）
2.mysql配置文件如下：

[client]
port            = 3306
socket          = /tmp/mysqld.sock
[mysqld]
user       = mysql
port            = 3306
socket          = /tmp/mysqld.sock
skip-external-locking
pid-file=/data/mysql/mysqld.pid
datadir = /data/mysql
log-error=/data/log/error.log
key_buffer_size = 3G
max_allowed_packet = 256M
thread_stack =1M
thread_cache_size = 32
sort_buffer_size = 2M
join_buffer_size =4M
read_buffer_size = 2M
net_buffer_length = 16M
read_rnd_buffer_size = 8M
myisam_sort_buffer_size = 64M
myisam_max_sort_file_size = 10G
myisam_repair_threads = 1
myisam-recover-options=DEFAULT
tmp_table_size = 256M
max_heap_table_size = 4G
myisam-recover  = BACKUP
max_connections =3000
max_connect_errors = 100000
query_cache_limit = 2M
query_cache_size =512M
query_cache_type = 1
query_cache_min_res_unit = 2k
default-storage-engine = Innodb
table_open_cache = 1000
transaction_isolation = READ-COMMITTED
innodb_additional_mem_pool_size = 16M
innodb_autoextend_increment = 8M
innodb_buffer_pool_size = 4G
innodb_checksums = 1
innodb_commit_concurrency = 0
innodb_concurrency_tickets = 500
innodb_doublewrite = 1
innodb_fast_shutdown = 1
innodb_file_io_threads = 4
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit = 0
innodb_flush_method = O_DIRECT
innodb_lock_wait_timeout = 30
innodb_locks_unsafe_for_binlog = 0
innodb_log_buffer_size = 3M
innodb_max_dirty_pages_pct = 80
innodb_open_files = 3000
innodb_data_file_path=ibdata1:50M:autoextend
innodb_log_file_size = 800M
innodb_log_files_in_group = 2
log_slow_queries  = /data/log/slow.log
long_query_time = 2
log-queries-not-using-indexes
log_bin_trust_function_creators = 1
expire_logs_days        = 10
max_binlog_size         = 100M
binlog_cache_size = 256K
binlog_stmt_cache_size=128K
max_binlog_cache_size = 512M
max_binlog_stmt_cache_size=512M
binlog_format=mixed
skip-name-resolve
server-id               = 1
log_bin                 = /data/mysql/mysql-bin.log
thread_concurrency= 32
innodb_thread_concurrency = 32
character-set-server = utf8
innodb_write_io_threads =32
innodb_read_io_threads= 32
innodb_max_dirty_pages_pct= 90
#wait_timeout=600
#interactive_timeout = 600
[mysqldump]
quick
max_allowed_packet = 16M
[mysql]
no-auto-rehash
[myisamchk]
key_buffer_size = 20M
sort_buffer_size = 20M
read_buffer = 2M
write_buffer = 2M
[mysqlhotcopy]
interactive-timeout

3.启动mysql进程
/etc/init.d/mysql start
至此，该项目完成部署，解析域名在VIP，即可。