Nginx支持反爬虫并限制客户端的请求的并发数

cat /usr/local/nginx/conf/agent_deny.conf

if ($http_user_agent ~* "qihoobot|Baiduspider|Googlebot|Googlebot-Mobile|Googlebot-Image|Mediapartners-Google|Adsbot-Google|Feedfetcher-Google|Yahoo! Slurp|Yahoo! Slurp China|YoudaoBot|Sosospider|Sogou spider|Sogou web spider|MSNBot|ia_archiver|Tomato Bot|Catall Spider|AcoiRobot") {
return 403;
}

if ($http_user_agent ~ "WinHttp|WebZIP|FetchURL|node-superagent|java/|FeedDemon|Jullo|JikeSpider|Indy Library|Alexa Toolbar|AskTbFXTV|AhrefsBot|CrawlDaddy|Java|Feedly|Apache-HttpAsyncClient|UniversalFeedParser|ApacheBench|Microsoft URL Control|Swiftbot|ZmEu|oBot|jaunty|Python-urllib|lightDeckReports Bot|YYSpider|DigExt|HttpClient|MJ12bot|heritrix|EasouSpider|Ezooms|BOT/0.1|YandexBot|FlightDeckReports|Linguee Bot|iaskspider^$") {
return 403;
}

if ($request_method !~ ^(GET|HEAD|POST)$) {
return 403;
}

if ($http_user_agent ~* (Python|Java|Wget|Scrapy|Curl|HttpClient|Spider)) {
return 403;
}

#屏蔽单个IP的命令是
#deny 123.45.6.7
#封整个段即从123.0.0.1到123.255.255.254的命令
#deny 123.0.0.0/8
#封IP段即从123.45.0.1到123.45.255.254的命令
#deny 124.45.0.0/16
#封IP段即从123.45.6.1到123.45.6.254的命令是
#deny 123.45.6.0/24
以下IP皆为流氓
deny 58.95.66.0/24;

注释：

一般情况下是允许百度爬虫和谷歌爬虫来爬取网站的内容的，例如网站官网的首页等，所以百度的爬虫和谷歌的爬虫是可以放开，允许来爬取网站内容的。
此文件agent_deny.conf 包含到网站官网的server虚拟主机里面的。

以下的nginx配置文件是方向代理负载均衡的配置文件：

server {
listen       80;
server_name  pk.tltest.com static.tltest.com;
access_log   /home/wwwlogs/access.log  main;

## 这个就是反爬虫文件
include /usr/local/nginx/conf/agent_deny.conf;
location / {
limit_req zone=reqip burst=200 nodelay;
proxy_cache cache_one;
proxy_cache_valid  200 304 301 302 99s;
proxy_cache_valid any 1s;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header REMOTE-HOST $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Connection "";
proxy_http_version 1.1;
proxy_next_upstream off;
proxy_ignore_client_abort on;
proxy_ignore_headers Set-Cookie Cache-Control;
client_max_body_size 30m;
client_body_buffer_size 256k;
proxy_connect_timeout 75;
proxy_send_timeout 300;
proxy_read_timeout 300;
proxy_buffer_size 1m;
proxy_buffers 8 512k;
proxy_busy_buffers_size 2m;
proxy_temp_file_write_size 2m;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503;
proxy_max_temp_file_size 128m;
proxy_pass http://backend; }

location *\.(php|python)$ {
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For  $remote_addr;
proxy_pass http://backend; }

####nginx前端限制客户端对网站某个目录的请求搜索的并发数
location = /novel/search {
limit_conn conip 2;
limit_req zone=reqip burst=3 nodelay;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For  $remote_addr;
proxy_pass http://backend; #access_log /home/wwwlogs/search.log  main;
}

####nginx前端限制客户端对网站某个目录的文件内容请求下载的并发数
location = /novel/read/cache {
limit_conn conip 1;
limit_req zone=reqip burst=2 nodelay;
limit_rate 512k;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For  $remote_addr;
proxy_pass http://backend; #access_log /home/wwwlogs/download.log  main;
}
####nginx前端限制客户端对网站某个目录的文件下apk下载的并发数
location = /novel/read/content {
limit_conn conip 5;
limit_req zone=reqip burst=10 nodelay;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For  $remote_addr;
proxy_pass http://backend; }

}

参考文档：
https://www.centos.bz/2018/01/nginx%E6%94%AF%E6%8C%81https%E5%B9%B6%E4%B8%94%E6%94%AF%E6%8C%81%E5%8F%8D%E7%88%AC%E8%99%AB/