nginx+confd+etcd的配置自动管理

一、配置文件

1、etcd配置数据：

/apps/svr/etcd/bin/etcdctl set /service_sgin/upstream/service1/server1 192.168.1.1
/apps/svr/etcd/bin/etcdctl set /service_sgin/subdomain/service1  "www.service1.com"

2、confd安装：

在github上下载：https://github.com/kelseyhightower/confd/releases 

在/etc/confd/下创建confd配置文件mkdir -p /etc/confd/{conf.d,templates}，具体配置如下：

[root@centos7-cloud ~]# cat /etc/confd/conf.d/nginx.conf.toml
[template]
prefix = "/service_sgin/"   #此配置对应etcd的默认目录
src = "nginx.conf.tmpl"    #对应templates下的模板文件
dest = '/apps/conf/nginx/nginx.conf'   #对应nginx的配置文件
keys = [
"/upstream",   #监控的键值目录
"/subdomain",  #监控的键值目录
]
#check_cmd = "/apps/svr/nginx/sbin/nginx -t -c {{.src}}"   #检测到变化后的测试
#reload_cmd = "/apps/svr/nginx/sbin/nginx -s reload"       #配置文件没问题后的reload

[root@centos7-cloud ~]#
[root@centos7-cloud ~]# cat /etc/confd/
conf.d/     confd.toml  templates/
[root@centos7-cloud ~]# cat /etc/confd/confd.toml
backend = "etcd"
confdir = "/etc/confd/"
interval = 5
#watch = true
log-level = "info"
nodes = [
"http://centos7-cloud:2379",
]
noop = false
prefix = "/"
scheme = "http"

[root@centos7-cloud ~]# cat /etc/confd/templates/nginx.conf.tmpl
user  apps apps;

worker_processes 24;
worker_cpu_affinity 000000000000000000000001;

error_log  /apps/logs/nginx/nginx_error.log  error;

#Specifies the value for maximum file descriptors that can be opened by this process.
worker_rlimit_nofile 65535;

events
{
use epoll;
worker_connections 65535;
}

http
{
include       mime.types;
#geoip_city /usr/local/nginx/conf/GeoLiteCity.dat;
#include geo.conf;
default_type  application/octet-stream;

#charset  gb2312;

sendfile on;
tcp_nopush     on;

client_body_timeout  5;
client_header_timeout 5;
keepalive_timeout  5;
send_timeout       5;

open_file_cache max=65535 inactive=20s;
open_file_cache_valid 30s;
open_file_cache_min_uses 1;

tcp_nodelay on;

fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 128k;

gzip on;
gzip_min_length  1k;
gzip_buffers     4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types       text/plain application/x-javascript text/css application/xml text/javascript application/json application/javascript;
gzip_vary on;

#limit_zone  crawler  $binary_remote_addr  10m;
proxy_temp_path   /dev/shm/temp;
proxy_cache_path  /dev/shm/cache levels=2:2:2   keys_zone=cache_go:200m inactive=5d max_size=7g;

log_format log_access "$remote_addr" "\t$remote_user" "\t$time_local" "\t$request" "\t$request_time" "\t$upstream_response_time" "\t$status" "\t$body_bytes_sent";

##############################################################################################################################################################

######################对应upstream的模板部分################
{{range $sub := ls "/subdomain"}}
upstream {{base $sub}} {
{{$subdir := printf "/upstream/%s/*" $sub}}{{range getvs $subdir}}
server {{.}}; {{end}}
keepalive_timeout 65s;
}
{{end}}

#####################对应server的模板部分####################
{{range gets "/subdomain/*"}}
server {
listen  80;
server_name  {{.Value}};
location / {
proxy_pass              http://{{base .Key}};
proxy_redirect  off;
proxy_set_header  Host                   $host;
proxy_set_header  X-Real-IP             $remote_addr;
proxy_set_header  X-Forwarded-For  $proxy_add_x_forwarded_for;
}
}
{{end}}
}

3、nginx安装配置，这里不多说

4、执行confd，生成配置：

[root@centos7-cloud bin]# ./confd -onetime -backend etcd -node http://127.0.0.1:2379 2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO Backend set to etcd
2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO Starting confd
2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO Backend nodes set to http://127.0.0.1:2379 2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO /apps/conf/nginx/nginx.conf has md5sum 391c4e48a2c484599896dbac3e0633c5 should be 7601c4d759f61e963075c38f532f5b36
2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO Target config /apps/conf/nginx/nginx.conf out of sync
2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO Target config /apps/conf/nginx/nginx.conf has been updated

[root@centos7-cloud bin]# ./confd -interval=60 -backend etcd -node http://127.0.0.1:2379 &
[1] 21501
[root@centos7-cloud bin]# 2017-06-17T15:40:27Z centos7-cloud ./confd[21501]: INFO Backend set to etcd
2017-06-17T15:40:27Z centos7-cloud ./confd[21501]: INFO Starting confd
2017-06-17T15:40:27Z centos7-cloud ./confd[21501]: INFO Backend nodes set to http://127.0.0.1:2379[/code] 
使用onetime参数的，配置文件生成一次后，confd程序就退出了，下面的那句，会每隔60秒轮询一次。一旦后端etcd相应的值发生变化就会重新生成相应的配置文件。

二、测试：

1、通过etcdctl命令，往etcd里增加k/v：

[root@centos7-cloud ~]# /apps/svr/etcd/bin/etcdctl set /service_sgin/upstream/service1/server3 192.168.1.3
192.168.1.3
[root@centos7-cloud ~]# /apps/svr/etcd/bin/etcdctl set /service_sgin/subdomain/service3  "www.service3.com"
www.service3.com

2、等待60s：

confd打印出来update更新的日志：

2017-06-17T15:41:27Z centos7-cloud ./confd[21501]: INFO /apps/conf/nginx/nginx.conf has md5sum 7601c4d759f61e963075c38f532f5b36 should be 5f0e6e8a0bb2fb8163883318094c9fb7
2017-06-17T15:41:27Z centos7-cloud ./confd[21501]: INFO Target config /apps/conf/nginx/nginx.conf out of sync
2017-06-17T15:41:27Z centos7-cloud ./confd[21501]: INFO Target config /apps/conf/nginx/nginx.conf has been updated
2017-06-17T15:42:27Z centos7-cloud ./confd[21501]: INFO /apps/conf/nginx/nginx.conf has md5sum 5f0e6e8a0bb2fb8163883318094c9fb7 should be 7da66fc7dd91dc65817744ea1fca23a8
2017-06-17T15:42:27Z centos7-cloud ./confd[21501]: INFO Target config /apps/conf/nginx/nginx.conf out of sync
2017-06-17T15:42:27Z centos7-cloud ./confd[21501]: INFO Target config /apps/conf/nginx/nginx.conf has been updated

3、对比文件：

初始：

自动更新后：