java web开发总结(三)：linux下超越apache + tomcat的 nginx + tomcat负载均衡配置

关于nginx，不了解的请先阅读这里：http://wiki.nginx.org/Main，中文文档：http://wiki.nginx.org/NginxChs
这是最近比较流行的一个轻量级的web server，在我的同机压力测试实验中，响应能力是apache的3倍以上。功能比apache简单，小巧，最大并发连接能到3万。

对于nginx的使用，我也是近期才开始，使用的目的是在不增加硬件的条件下提升服务响应能力，以下配置适用环境：
linux + nginx0.7.64+ tomcat6

nginx同时是一个很强的反向代理server，可以用来跟tomcat做负载均衡集群，而且配置很简单，不过有个需要注意的地方：目前nginx跟后端server使用http1.0协议，不能keepalive，只能每次重新打开连接。需要关闭后端的keepalive，否则会产生大量后端的timewait状态的socket连接。

1 nginx代理tomcat关键配置代码如下：

Xml代码

server {

listen 80;

server_name YOUR_DOMAIN;

root /PATH/TO/YOUR/WEB/APPLICATION;

location / {

index.jsp;

}

location ~ ^/WEB-INF/* {#禁止访问被保护的目录

deny all;

}

location ~ /.do$ {

proxy_pass http://localhost:8080;#转发给tomcat端口，也可以是任何server。
proxy_set_header X-Real-IP $remote_addr;#转发客户端真实IP

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

location ~ /.jsp$ {

proxy_pass http://localhost:8080;
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

location ~ ^/servlet/* {

proxy_pass http://localhost:8080;
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

}

server {
listen          80;
server_name     YOUR_DOMAIN;
root            /PATH/TO/YOUR/WEB/APPLICATION;
location / {
index.jsp;
}
location ~ ^/WEB-INF/* {#禁止访问被保护的目录
deny all;
}
location ~ /.do$ {
proxy_pass              http://localhost:8080;#转发给tomcat端口，也可以是任何server。 proxy_set_header        X-Real-IP $remote_addr;#转发客户端真实IP
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}
location ~ /.jsp$ {
proxy_pass              http://localhost:8080; proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}
location ~ ^/servlet/* {
proxy_pass              http://localhost:8080; proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}
}

2 nginx负载均衡核心配置代码：

Xml代码

http {

upstream myproject {

server 127.0.0.1:8000 weight=3;

server 127.0.0.1:8001;

server 127.0.0.1:8002;

server 127.0.0.1:8003;

}

server {

listen 80;

server_name www.domain.com;

location / {

proxy_pass http://myproject;#myproject 前面要加http://，否则异常

}

}

}

http {
upstream myproject {
server 127.0.0.1:8000 weight=3;
server 127.0.0.1:8001;
server 127.0.0.1:8002;
server 127.0.0.1:8003;
}
server {
listen 80;
server_name www.domain.com;
location / {
proxy_pass http://myproject;#myproject 前面要加http://，否则异常
}
}
}

3 nginx+tomcat负载均衡简单配置：不用修改任何tomcat配置，只要tomcat开启http1.1协议端口(tomcat默认开启)

Xml代码

http {

upstream myproject {

server 127.0.0.1:8000 weight=3;

server 127.0.0.1:8001;

server 127.0.0.1:8002;

server 127.0.0.1:8003;

}

server {

listen 80;

server_name www.domain.com;

root /PATH/TO/YOUR/WEB/APPLICATION;

location / {

index.jsp;

}

location ~ ^/WEB-INF/* {#禁止访问被保护的目录

deny all;

}

location ~ /.do$ {

proxy_pass http://myproject;#转发给tomcat端口，也可以是任何server。
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

location ~ /.jsp$ {

proxy_pass http://myproject;#myproject可以任意自定义，但是一定要在前面加http://
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

location ~ ^/servlets/* {

proxy_pass http://myproject;
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

}

}

http {
upstream myproject {
server 127.0.0.1:8000 weight=3;
server 127.0.0.1:8001;
server 127.0.0.1:8002;
server 127.0.0.1:8003;
}
server {
listen 80;
server_name www.domain.com;
root /PATH/TO/YOUR/WEB/APPLICATION;
location / {
index.jsp;
}
location ~ ^/WEB-INF/* {#禁止访问被保护的目录
deny all;
}
location ~ /.do$ {
proxy_pass              http://myproject;#转发给tomcat端口，也可以是任何server。 proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}
location ~ /.jsp$ {
proxy_pass              http://myproject;#myproject可以任意自定义，但是一定要在前面加http:// proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}
location ~ ^/servlets/* {
proxy_pass              http://myproject; proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}
}
}

4 实际环境复杂配置：tomcat动态负载均衡 + 静态文件

Xml代码

worker_processes 8;

worker_rlimit_nofile 65535;

events {

use epoll;#开启epoll模式

worker_connections 65535;

}

http {

include mime.types;

default_type application/octet-stream;

#log_format main '$remote_addr - $remote_user [$time_local] "$request" '

# '$status $body_bytes_sent "$http_referer" '

# '"$http_user_agent" "$http_x_forwarded_for"';

#access_log logs/access.log main;

sendfile on;#linux下强大的静态文件发送功能，一定要开启

tcp_nopush on;

tcp_nodelay on;

#keepalive_timeout 0;

keepalive_timeout 5;

keepalive_requests 10;

server_names_hash_bucket_size 128;

client_header_timeout 10;

client_body_timeout 10;

send_timeout 10;

client_header_buffer_size 1k;

large_client_header_buffers 4 4k;

client_max_body_size 8m;

gzip on;#开启gzip，节省带宽

gzip_min_length 1100;

gzip_buffers 4 8k;

gzip_types text/xml;

output_buffers 1 32k;

postpone_output 1460;

limit_rate_after 3m;#限速模块，前3M下载时不限速

limit_rate 512k; #限速模块

upstream statproxy {#负载均衡模块，核心配置

server 127.0.0.1:8080;

server 127.0.0.1:8180;

}

server {#纯静态文件访问

listen 80;

server_name a.udomain.com;

index index.html index.php;

root /PATH/TO/YOUR/WEB/APPLICATIONA;

#charset koi8-r;

#access_log logs/host.access.log main;

#error_page 404 /404.html;

# redirect server error pages to the static page /50x.html

#

error_page 500 502 503 504 /50x.html;

location = /50x.html {

root html;

}

location ~ ^/server-status/ {#服务器状态监控，可以通过：http://a.udomain.com/server-status/访问，需要在编译安装是添加此模块，默认不带。

stub_status on;

access_log off;

}

}

server {#动态负载均衡 + 动态静态分离

listen 80;

server_name b.udomain.com;

index index.html index.php;

root /PATH/TO/YOUR/WEB/APPLICATIONB;

error_page 500 502 503 504 /50x.html;

location ~ ^/WEB-INF/* {#禁止访问被保护的目录

deny all;

}

location ~ /.do$ {

proxy_pass http://statproxy;#经负载均衡模块转发
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

location ~ /.jsp$ {

proxy_pass http://statproxy;
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

location ~ ^/servlet/* {

proxy_pass http://statproxy;
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

}

server {#动态无负载均衡+静态访问

listen 80;

server_name c.udomain.com;

index index.html index.php;

root /PATH/TO/YOUR/WEB/APPLICATIONC;

#charset koi8-r;

#access_log logs/host.access.log main;

#error_page 404 /404.html;

# redirect server error pages to the static page /50x.html

#

error_page 500 502 503 504 /50x.html;

location = /50x.html {

root html;

}

location ~ ^/WEB-INF/* {#禁止访问被保护的目录

deny all;

}

location ~ /.jsp$ {

proxy_pass http://localhost:8280;#直接转发
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

location ~ /.do$ {

proxy_pass http://localhost:8280;
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

location ~ ^/servlet/* {

proxy_pass http://localhost:8280;
proxy_set_header X-Real-IP $remote_addr;

proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_set_header Host $http_host;

}

}

}

worker_processes  8;
worker_rlimit_nofile 65535;
events {
use epoll;#开启epoll模式
worker_connections  65535;
}
http {
include        mime.types;
default_type  application/octet-stream;

#log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
#                 '$status $body_bytes_sent "$http_referer" '
#                 '"$http_user_agent" "$http_x_forwarded_for"';
#access_log  logs/access.log  main;

sendfile        on;#linux下强大的静态文件发送功能，一定要开启
tcp_nopush     on;
tcp_nodelay on;

#keepalive_timeout  0;
keepalive_timeout  5;
keepalive_requests 10;
server_names_hash_bucket_size 128;

client_header_timeout  10;
client_body_timeout    10;
send_timeout           10;

client_header_buffer_size    1k;
large_client_header_buffers  4 4k;
client_max_body_size 8m;

gzip on;#开启gzip，节省带宽
gzip_min_length  1100;
gzip_buffers     4 8k;
gzip_types        text/xml;

output_buffers   1 32k;
postpone_output  1460;

limit_rate_after 3m;#限速模块，前3M下载时不限速
limit_rate 512k; #限速模块

upstream  statproxy  {#负载均衡模块，核心配置
server   127.0.0.1:8080;
server   127.0.0.1:8180;
}

server {#纯静态文件访问
listen        80;
server_name  a.udomain.com;
index index.html index.php;
root  /PATH/TO/YOUR/WEB/APPLICATIONA;
#charset koi8-r;
#access_log  logs/host.access.log  main;
#error_page  404              /404.html;
# redirect server error pages to the static page /50x.html
#
error_page   500 502 503 504  /50x.html;

location = /50x.html {
root   html;
}

location ~ ^/server-status/ {#服务器状态监控，可以通过：http://a.udomain.com/server-status/访问，需要在编译安装是添加此模块，默认不带。
stub_status on;
access_log off;
}
}

server {#动态负载均衡 + 动态静态分离
listen        80;
server_name  b.udomain.com;
index index.html index.php;
root  /PATH/TO/YOUR/WEB/APPLICATIONB;
error_page   500 502 503 504  /50x.html;
location ~ ^/WEB-INF/* {#禁止访问被保护的目录
deny all;
}
location ~ /.do$ {
proxy_pass              http://statproxy;#经负载均衡模块转发 proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}

location ~ /.jsp$ {
proxy_pass              http://statproxy; proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}

location ~ ^/servlet/* {
proxy_pass              http://statproxy; proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}
}

server {#动态无负载均衡+静态访问
listen        80;
server_name  c.udomain.com;
index index.html index.php;
root  /PATH/TO/YOUR/WEB/APPLICATIONC;
#charset koi8-r;
#access_log  logs/host.access.log  main;
#error_page  404              /404.html;
# redirect server error pages to the static page /50x.html
#
error_page   500 502 503 504  /50x.html;
location = /50x.html {
root   html;
}

location ~ ^/WEB-INF/* {#禁止访问被保护的目录
deny all;
}

location ~ /.jsp$ {
proxy_pass              http://localhost:8280;#直接转发 proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}

location ~ /.do$ {
proxy_pass              http://localhost:8280; proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}

location ~ ^/servlet/* {
proxy_pass              http://localhost:8280; proxy_set_header        X-Real-IP $remote_addr;
proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header        Host $http_host;
}
}
}

5 tomcat端配置：
默认开启http1.1协议即可。建议使用NIO连接器。关于NIO，可以参考tomcat官方文档：http://tomcat.apache.org/tomcat-6.0-doc/config/http.html

注意问题：
nginx+tomcat下：
request.getRemoteAddr()会取不到正确的ip地址的，但是
proxy_set_header X-Real-IP $remote_addr;
这个配置把正确的ip地址放在了header里了，可以通过：
String ip = request.getHeader("X-Real-IP");
获取到正确的用户ip地址，当然这样会需要修改程序，的确不是个好的办法，不能适用所有场景。

参考文档：
nginx文档：http://wiki.nginx.org/Main，中文文档：http://wiki.nginx.org/NginxChs
nginx+tomcat配置示例：http://wiki.nginx.org/NginxJavaServers
nginx负载均衡配置示例：http://wiki.nginx.org/NginxLoadBalanceExample
tomcat nio配置参数文档：http://tomcat.apache.org/tomcat-6.0-doc/config/http.html