006.Kubernetes二进制部署ETCD

一 部署ETCD集群

1.1 安装ETCD

etcd 是基于 Raft 的分布式 key-value 存储系统，由 CoreOS 开发，常用于服务发现、共享配置以及并发控制（如 leader 选举、分布式锁等）。kubernetes 使用 etcd 存储所有运行数据。

1 etcd 是基于 Raft 的分布式 key-value 存储系统，由 CoreOS 开发，常用于服务发现、共享配置以及并发控制（如 leader 选举、分布式锁等）。kubernetes 使用 etcd 存储所有运行数据。
2 [root@k8smaster01 ~]# cd /opt/k8s/work
3 [root@k8smaster01 work]# wget https://github.com/coreos/etcd/releases/download/v3.3.13/etcd-v3.3.13-linux-amd64.tar.gz
4 [root@k8smaster01 work]# tar -xvf etcd-v3.3.13-linux-amd64.tar.gz

1.2 分发ETCD

1 [root@k8smaster01 ~]# cd /opt/k8s/work
2 [root@k8smaster01 work]# source /opt/k8s/bin/environment.sh
3 [root@k8smaster01 work]# for master_ip in ${MASTER_IPS[@]}
4   do
5     echo ">>> ${master_ip}"
6     scp etcd-v3.3.13-linux-amd64/etcd* root@${master_ip}:/opt/k8s/bin
7     ssh root@${master_ip} "chmod +x /opt/k8s/bin/*"
8   done

1.3 创建etcd证书和密钥

1 [root@k8smaster01 ~]# cd /opt/k8s/work
2 [root@k8smaster01 work]# cat > etcd-csr.json <<EOF
3 {
4     "CN": "etcd",
5     "hosts": [
6     "127.0.0.1",
7     "172.24.8.71",
8     "172.24.8.72",
9     "172.24.8.73"
10   ],
11     "key": {
12         "algo": "rsa",
13         "size": 2048
14     },
15     "names": [
16         {
17             "C": "CN",
18             "ST": "Shanghai",
19             "L": "Shanghai",
20             "O": "k8s",
21             "OU": "System"
22         }
23     ]
24 }
25 EOF
26 #创建etcd的CA证书请求文件

解释： hosts 字段指定授权使用该证书的 etcd 节点 IP 或域名列表，需要将 etcd 集群的三个节点 IP 都列在其中。

1 [root@k8smaster01 ~]# cd /opt/k8s/work
2 [root@k8smaster01 work]# cfssl gencert -ca=/opt/k8s/work/ca.pem \
3 -ca-key=/opt/k8s/work/ca-key.pem -config=/opt/k8s/work/ca-config.json \
4 -profile=kubernetes etcd-csr.json | cfssljson -bare etcd	#生成CA密钥（ca-key.pem）和证书（ca.pem）

1.4 分发证书和私钥

1 [root@k8smaster01 ~]# cd /opt/k8s/work
2 [root@k8smaster01 work]# source /opt/k8s/bin/environment.sh
3 [root@k8smaster01 work]# for master_ip in ${MASTER_IPS[@]}
4   do
5     echo ">>> ${master_ip}"
6     ssh root@${master_ip} "mkdir -p /etc/etcd/cert"
7     scp etcd*.pem root@${master_ip}:/etc/etcd/cert/
8   done

1.5 创建etcd的systemd

1 [root@k8smaster01 ~]# cd /opt/k8s/work
2 [root@k8smaster01 work]# source /opt/k8s/bin/environment.sh
3 [root@k8smaster01 work]# cat > etcd.service.template <<EOF
4 [Unit]
5 Description=Etcd Server
6 After=network.target
7 After=network-online.target
8 Wants=network-online.target
9 Documentation=https://github.com/coreos
10
11 [Service]
12 Type=notify
13 WorkingDirectory=${ETCD_DATA_DIR}
14 ExecStart=/opt/k8s/bin/etcd \\
15   --data-dir=${ETCD_DATA_DIR} \\
16   --wal-dir=${ETCD_WAL_DIR} \\
17   --name=##MASTER_NAME## \\
18   --cert-file=/etc/etcd/cert/etcd.pem \\
19   --key-file=/etc/etcd/cert/etcd-key.pem \\
20   --trusted-ca-file=/etc/kubernetes/cert/ca.pem \\
21   --peer-cert-file=/etc/etcd/cert/etcd.pem \\
22   --peer-key-file=/etc/etcd/cert/etcd-key.pem \\
23   --peer-trusted-ca-file=/etc/kubernetes/cert/ca.pem \\
24   --peer-client-cert-auth \\
25   --client-cert-auth \\
26   --listen-peer-urls=https://##MASTER_IP##:2380 \\
27   --initial-advertise-peer-urls=https://##MASTER_IP##:2380 \\
28   --listen-client-urls=https://##MASTER_IP##:2379,http://127.0.0.1:2379 \\
29   --advertise-client-urls=https://##MASTER_IP##:2379 \\
30   --initial-cluster-token=etcd-cluster-0 \\
31   --initial-cluster=${ETCD_NODES} \\
32   --initial-cluster-state=new \\
33   --auto-compaction-mode=periodic \\
34   --auto-compaction-retention=1 \\
35   --max-request-bytes=33554432 \\
36   --quota-backend-bytes=6442450944 \\
37   --heartbeat-interval=250 \\
38   --election-timeout=2000
39 Restart=on-failure
40 RestartSec=5
41 LimitNOFILE=65536
42
43 [Install]
44 WantedBy=multi-user.target
45 EOF

解释： WorkingDirectory、--data-dir：指定工作目录和数据目录为 ${ETCD_DATA_DIR}，需在启动服务前创建这个目录； --wal-dir：指定 wal 目录，为了提高性能，一般使用 SSD 或者和 --data-dir 不同的磁盘； --name：指定节点名称，当 --initial-cluster-state 值为 new 时，--name 的参数值必须位于 --initial-cluster 列表中； --cert-file、--key-file：etcd server 与 client 通信时使用的证书和私钥； --trusted-ca-file：签名 client 证书的 CA 证书，用于验证 client 证书； --peer-cert-file、--peer-key-file：etcd 与 peer 通信使用的证书和私钥； --peer-trusted-ca-file：签名 peer 证书的 CA 证书，用于验证 peer 证书。

1.6 修改systemd相应地址

1 [root@k8smaster01 ~]# cd /opt/k8s/work
2 [root@k8smaster01 work]# source /opt/k8s/bin/environment.sh
3 [root@k8smaster01 work]# for (( i=0; i < 3; i++ ))
4   do
5     sed -e "s/##MASTER_NAME##/${MASTER_NAMES[i]}/" -e "s/##MASTER_IP##/${MASTER_IPS[i]}/" etcd.service.template > etcd-${MASTER_IPS[i]}.service
6   done

1.7 分发etcd systemd

1 [root@k8smaster01 ~]# cd /opt/k8s/work
2 [root@k8smaster01 work]# source /opt/k8s/bin/environment.sh
3 [root@k8smaster01 work]# for master_ip in ${MASTER_IPS[@]}
4   do
5     echo ">>> ${master_ip}"
6     scp etcd-${master_ip}.service root@${master_ip}:/etc/systemd/system/etcd.service
7   done

二 启动并验证

2.1 启动ETCD

1 [root@k8smaster01 ~]# cd /opt/k8s/work
2 [root@k8smaster01 work]# source /opt/k8s/bin/environment.sh
3 [root@k8smaster01 work]# for master_ip in ${MASTER_IPS[@]}
4   do
5     echo ">>> ${master_ip}"
6     ssh root@${master_ip} "mkdir -p ${ETCD_DATA_DIR} ${ETCD_WAL_DIR}"
7     ssh root@${master_ip} "systemctl daemon-reload && systemctl enable etcd && systemctl restart etcd " &
8   done

2.2 检查ETCD启动

1 [root@k8smaster01 ~]# cd /opt/k8s/work
2 [root@k8smaster01 work]# source /opt/k8s/bin/environment.sh
3 [root@k8smaster01 work]# for master_ip in ${MASTER_IPS[@]}
4   do
5     echo ">>> ${master_ip}"
6     ssh root@${master_ip} "systemctl status etcd|grep Active"
7   done

2.3 验证服务状态

1 [root@k8smaster01 ~]# cd /opt/k8s/work
2 [root@k8smaster01 work]# source /opt/k8s/bin/environment.sh
3 [root@k8smaster01 work]# for master_ip in ${MASTER_IPS[@]}
4   do
5     echo ">>> ${master_ip}"
6     ETCDCTL_API=3 /opt/k8s/bin/etcdctl \
7     --endpoints=https://${master_ip}:2379 \
8     --cacert=/etc/kubernetes/cert/ca.pem \
9     --cert=/etc/etcd/cert/etcd.pem \
10     --key=/etc/etcd/cert/etcd-key.pem endpoint health
11   done

2.4 查看ETCD当前leader

1 [root@k8smaster01 ~]# source /opt/k8s/bin/environment.sh
2 [root@k8smaster01 ~]# ETCDCTL_API=3 /opt/k8s/bin/etcdctl \
3   -w table --cacert=/etc/kubernetes/cert/ca.pem \
4   --cert=/etc/etcd/cert/etcd.pem \
5   --key=/etc/etcd/cert/etcd-key.pem \
6   --endpoints=${ETCD_ENDPOINTS} endpoint status

如上所示，当前ETCD集群的leader为172.24.8.71。