kubernetes v1.18.2 二进制 双栈 etcd 部署

说明：

# K8S 组件之间连接使用IPV6进行通信包括etcd
# 设置feature-gates IPv6DualStack=true 所有组件
# 证书包括IPV6 IPV4 IP 集群可以IPV6 也可以IPv4 进行通信

配置环境变量文件

# 创建任意目录
mkdir -p ipv6
cd ipv6
# 创建环境变量文件
cat << EOF | tee environment.sh
#!/bin/bash
# 设置证书环境变量
# 设置证书使用时间87600h 10年
export EXPIRY_TIME="87600h"
# 签发证书IP
export ETCD_MEMBER_1_IP="192.168.2.175"
export ETCD_MEMBER_2_IP="192.168.2.176"
export ETCD_MEMBER_3_IP="192.168.2.177"
# 机器名
export ETCD_MEMBER_1_HOSTNAMES="k8s-master-1"
export ETCD_MEMBER_2_HOSTNAMES="k8s-master-2"
export ETCD_MEMBER_3_HOSTNAMES="k8s-master-3"
#etcd IPV6 地址
ETCD_MEMBER_1_IP6="fc00:bd4:efa8:1001:5054:ff:fe49:9888"
ETCD_MEMBER_2_IP6="fc00:bd4:efa8:1001:5054:ff:fe47:357b"
ETCD_MEMBER_3_IP6="fc00:bd4:efa8:1001:5054:ff:fec6:74fb"
# etcd 集群通讯证书
export ETCD_SERVER_HOSTNAMES="\"\${ETCD_MEMBER_1_HOSTNAMES}\",\"\${ETCD_MEMBER_2_HOSTNAMES}\",\"\${ETCD_MEMBER_3_HOSTNAMES}\""
export ETCD_SERVER_IPS="\"\${ETCD_MEMBER_1_IP}\",\"\${ETCD_MEMBER_2_IP}\",\"\${ETCD_MEMBER_3_IP}\",\"\${ETCD_MEMBER_1_IP6}\",\"\${ETCD_MEMBER_2_IP6}\",\"\${ETCD_MEMBER_3_IP6}\""
#证书所需要的配置参数
export CERT_ST="GuangDong"
export CERT_L="GuangZhou"
export CERT_O="k8s"
export CERT_OU="Qist"
export CERT_PROFILE="kubernetes"
# 设置工作目录
export HOST_PATH=\`pwd\`
# kube-apiserver 服务器IP 如果外部访问K8s 集群使用VIP ip 请在下面添加vip ip
export K8S_APISERVER_VIPA="\"192.168.2.175\",\"192.168.2.176\",\"192.168.2.177\""
export K8S_APISERVER_VIP="\"fc00:bd4:efa8:1001:5054:ff:fe49:9888\",\"fc00:bd4:efa8:1001:5054:ff:fe47:357b\",\"fc00:bd4:efa8:1001:5054:ff:fec6:74fb\",\${K8S_APISERVER_VIPA}"
# kubernetes 服务 IP (一般是 SERVICE_CIDR 中第一个IP)
export CLUSTER_KUBERNETES_SVC_IP="\"8888:8000::1\",\"10.66.0.1\""
# 设置集群参数
export CLUSTER_NAME=kubernetes
# kubectl 访问url地址
export KUBE_API=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:5443
# K8S 外部IP 这里高可用使用本地环回IP
export K8S_VIP_DOMAIN=::1
export K8S_SSL="\"${K8S_VIP_DOMAIN}\",\"127.0.0.1\""
# 生成 EncryptionConfig 所需的加密 key
export ENCRYPTION_KEY=\$(head -c 32 /dev/urandom | base64)
# 设置连接KUBE_APISERVER ip
export KUBE_APISERVER=https://[::1]:5443
# kubelet kube-proxy 连接集群所用url
export KUBE_API_KUBELET=https://[::1]:6443
# 创建bootstrap配置
export TOKEN_ID=\$(head -c 16 /dev/urandom | od -An -t x | tr -dc a-f3-9|cut -c 1-6)
export TOKEN_SECRET=\$(head -c 16 /dev/urandom | md5sum | head -c 16)
export BOOTSTRAP_TOKEN=\${TOKEN_ID}.${TOKEN_SECRET}
#集群域名
export CLUSTER_DNS_DOMAIN="cluster.local"
#集群DNS
export CLUSTER_DNS_SVC_IP="8888:8000::2"
EOF
# 生效环境变量
source  ./environment.sh

服务器相关设置

ssh 192.168.2.175 hostnamectl set-hostname k8s-master-1
ssh 192.168.2.176 hostnamectl set-hostname k8s-master-2
ssh 192.168.2.177 hostnamectl set-hostname k8s-master-3
ssh 192.168.2.185 hostnamectl set-hostname k8s-node-1
ssh 192.168.2.187 hostnamectl set-hostname k8s-node-2

设置关闭防火墙及SELINUX

# centosx
sed -i 's/SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config
systemctl stop firewalld && systemctl disable firewalld
setenforce 0
# Ubuntu
systemctl stop ufw.service
systemctl disable ufw.service

安装及配置CFSSL 签发证书使用

#go 环境部署
yum install go
vi ~/.bash_profile
GOBIN=/root/go/bin/
PATH=$PATH:$GOBIN:$HOME/bin
export PATH
go get  github.com/cloudflare/cfssl/cmd/cfssl
go get  github.com/cloudflare/cfssl/cmd/cfssljson

生成etcd 相关证书

# 创建etcd K8S 证书json 存放目录
mkdir -p ${HOST_PATH}/cfssl/{k8s,etcd}
# 创建签发证书存放目录
mkdir -p ${HOST_PATH}/cfssl/pki/{k8s,etcd}
# CA 配置文件用于配置根证书的使用场景 (profile) 和具体参数 (usage，过期时间、服务端认证、客户端认证、加密等)，后续在签名其它证书时需要指定特定场景。
cat << EOF | tee ${HOST_PATH}/cfssl/ca-config.json
{
"signing": {
"default": {
"expiry": "${EXPIRY_TIME}"
},
"profiles": {
"${CERT_PROFILE}": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "${EXPIRY_TIME}"
}
}
}
}
EOF
# 创建 ETCD CA 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-ca-csr.json
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
],
"ca": {
"expiry": "${EXPIRY_TIME}"
}
}
EOF
# etcd ca 证书签发
cfssl gencert -initca ${HOST_PATH}/cfssl/etcd/etcd-ca-csr.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-ca
# 创建 ETCD Server 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-server.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"::1",
${ETCD_SERVER_IPS},
${ETCD_SERVER_HOSTNAMES}
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 ETCD Server 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/etcd/etcd-server.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-server
# 创建 ETCD Member 1 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"::1",
"${ETCD_MEMBER_1_IP}",
"${ETCD_MEMBER_1_IP6}",
"${ETCD_MEMBER_1_HOSTNAMES}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 ETCD Member 1 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_1_HOSTNAMES}
# 创建 ETCD Member 2 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"::1",
"${ETCD_MEMBER_2_IP}",
"${ETCD_MEMBER_2_IP6}",
"${ETCD_MEMBER_2_HOSTNAMES}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 ETCD Member 2 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_2_HOSTNAMES}
# 创建 ETCD Member 3 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"::1",
"${ETCD_MEMBER_3_IP}",
"${ETCD_MEMBER_3_IP6}",
"${ETCD_MEMBER_3_HOSTNAMES}"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 ETCD Member 3 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_3_HOSTNAMES}
# 创建 ETCD Client 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-client.json
{
"CN": "client",
"hosts": [""],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "$CERT_ST",
"L": "$CERT_L",
"O": "$CERT_O",
"OU": "$CERT_OU"
}
]
}
EOF
# 生成 ETCD Client 证书和私钥
cfssl gencert \
-ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem \
-ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem \
-config=${HOST_PATH}/cfssl/ca-config.json \
-profile=${CERT_PROFILE} \
${HOST_PATH}/cfssl/etcd/etcd-client.json | \
cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-client
# 分发生成的证书到所有需要部署etcd 节点
ssh 192.168.2.175 mkdir -p /apps/etcd/ssl
ssh 192.168.2.176 mkdir -p /apps/etcd/ssl
ssh 192.168.2.177 mkdir -p /apps/etcd/ssl
# 分发文件
scp -r ./cfssl/pki/etcd/* 192.168.2.175:/apps/etcd/ssl/
scp -r ./cfssl/pki/etcd/* 192.168.2.176:/apps/etcd/ssl/
scp -r ./cfssl/pki/etcd/* 192.168.2.177:/apps/etcd/ssl/

etcd 二进制文件准备

wget https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-linux-amd64.tar.gz
# 解压下载好文件
tar -xvf etcd-v3.4.7-linux-amd64.tar.gz
# 创建二进制远程存放目录
ssh 192.168.2.175 mkdir -p /apps/etcd/bin
ssh 192.168.2.176 mkdir -p /apps/etcd/bin
ssh 192.168.2.177 mkdir -p /apps/etcd/bin
# 分发解压好二进制文件
cd etcd-v3.4.7-linux-amd64/
scp -r etcd* 192.168.2.175:/apps/etcd/bin
scp -r etcd* 192.168.2.176:/apps/etcd/bin
scp -r etcd* 192.168.2.177:/apps/etcd/bin

etcd 配置文件准备

# 创建配置文件存放目录
ssh 192.168.2.175 mkdir -p /apps/etcd/conf
ssh 192.168.2.176 mkdir -p /apps/etcd/conf
ssh 192.168.2.177 mkdir -p /apps/etcd/conf
# 192.168.2.175   配置
ssh 192.168.2.175
cat << EOF | tee /apps/etcd/conf/etcd
ETCD_OPTS="--name=k8s-master-1 \\
--data-dir=/apps/etcd/data/default.etcd \\
--wal-dir=/apps/etcd/data/default.etcd/wal \\
--listen-peer-urls=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2380 \\
--listen-client-urls=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2379,https://[::1]:2379 \\
--advertise-client-urls=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2379 \\
--initial-advertise-peer-urls=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2380 \\
--initial-cluster=k8s-master-1=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2380,k8s-master-2=https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2380,k8s-master-3=https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2380 \\
--initial-cluster-token=k8s-master-1=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2380,k8s-master-2=https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2380,k8s-master-3=https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2380 \\
--initial-cluster-state=new \\
--heartbeat-interval=6000 \\
--election-timeout=30000 \\
--snapshot-count=5000 \\
--auto-compaction-retention=1 \\
--max-request-bytes=33554432 \\
--quota-backend-bytes=17179869184 \\
--trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \\
--cert-file=/apps/etcd/ssl/etcd-server.pem \\
--key-file=/apps/etcd/ssl/etcd-server-key.pem \\
--peer-cert-file=/apps/etcd/ssl/etcd-member-k8s-master-1.pem \\
--peer-key-file=/apps/etcd/ssl/etcd-member-k8s-master-1-key.pem \\
--peer-client-cert-auth \\
--enable-v2=true \\
--peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem"
EOF
# 192.168.2.176 配置
ssh 192.168.2.176
cat << EOF | tee /apps/etcd/conf/etcd
ETCD_OPTS="--name=k8s-master-2 \\
--data-dir=/apps/etcd/data/default.etcd \\
--wal-dir=/apps/etcd/data/default.etcd/wal \\
--listen-peer-urls=https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2380 \\
--listen-client-urls=https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2379,https://[::1]:2379 \\
--advertise-client-urls=https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2379 \\
--initial-advertise-peer-urls=https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2380 \\
--initial-cluster=k8s-master-1=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2380,k8s-master-2=https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2380,k8s-master-3=https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2380 \\
--initial-cluster-token=k8s-master-1=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2380,k8s-master-2=https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2380,k8s-master-3=https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2380 \\
--initial-cluster-state=new \\
--heartbeat-interval=6000 \\
--election-timeout=30000 \\
--snapshot-count=5000 \\
--auto-compaction-retention=1 \\
--max-request-bytes=33554432 \\
--quota-backend-bytes=17179869184 \\
--trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \\
--cert-file=/apps/etcd/ssl/etcd-server.pem \\
--key-file=/apps/etcd/ssl/etcd-server-key.pem \\
--peer-cert-file=/apps/etcd/ssl/etcd-member-k8s-master-2.pem \\
--peer-key-file=/apps/etcd/ssl/etcd-member-k8s-master-2-key.pem \\
--peer-client-cert-auth \\
--enable-v2=true \\
--peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem"
EOF
# 192.168.2.177 配置
ssh 192.168.2.177
cat << EOF | tee /apps/etcd/conf/etcd
ETCD_OPTS="--name=k8s-master-3 \\
--data-dir=/apps/etcd/data/default.etcd \\
--wal-dir=/apps/etcd/data/default.etcd/wal \\
--listen-peer-urls=https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2380 \\
--listen-client-urls=https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2379,https://[::1]:2379 \\
--advertise-client-urls=https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2379 \\
--initial-advertise-peer-urls=https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2380 \\
--initial-cluster=k8s-master-1=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2380,k8s-master-2=https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2380,k8s-master-3=https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2380 \\
--initial-cluster-token=k8s-master-1=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2380,k8s-master-2=https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2380,k8s-master-3=https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2380 \\
--initial-cluster-state=new \\
--heartbeat-interval=6000 \\
--election-timeout=30000 \\
--snapshot-count=5000 \\
--auto-compaction-retention=1 \\
--max-request-bytes=33554432 \\
--quota-backend-bytes=17179869184 \\
--trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \\
--cert-file=/apps/etcd/ssl/etcd-server.pem \\
--key-file=/apps/etcd/ssl/etcd-server-key.pem \\
--peer-cert-file=/apps/etcd/ssl/etcd-member-k8s-master-3.pem \\
--peer-key-file=/apps/etcd/ssl/etcd-member-k8s-master-3-key.pem \\
--peer-client-cert-auth \\
--enable-v2=true \\
--peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem"
EOF

etcd 启动文件配置

cat << EOF | tee etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/etcd-io/etcd
[Service]
Type=notify
LimitNOFILE=65535
LimitNPROC=65535
LimitCORE=infinity
LimitMEMLOCK=infinity
User=etcd
Group=etcd
WorkingDirectory=/apps/etcd/data/default.etcd
EnvironmentFile=-/apps/etcd/conf/etcd
ExecStart=/apps/etcd/bin/etcd \$ETCD_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
# 上传启动文件到服务器
scp etcd.service 192.168.2.175:/usr/lib/systemd/system
scp etcd.service 192.168.2.176:/usr/lib/systemd/system
scp etcd.service 192.168.2.176:/usr/lib/systemd/system

etcd 启动准备

# 创建etcd 用户
ssh  192.168.2.175 useradd etcd -s /sbin/nologin -M
ssh  192.168.2.176 useradd etcd -s /sbin/nologin -M
ssh  192.168.2.177 useradd etcd -s /sbin/nologin -M
# 创建etcd 存储文件目录
ssh  192.168.2.175 mkdir -p /apps/etcd/data/default.etcd/wal
ssh  192.168.2.176 mkdir -p /apps/etcd/data/default.etcd/wal
ssh  192.168.2.177 mkdir -p /apps/etcd/data/default.etcd/wal
# 给/apps/etcd etcd 用户权限
ssh  192.168.2.175 chown -R etcd:etcd /apps/etcd/
ssh  192.168.2.176 chown -R etcd:etcd /apps/etcd/
ssh  192.168.2.177 chown -R etcd:etcd /apps/etcd/

etcd 启动

# 刷新service
ssh 192.168.2.175 systemctl daemon-reload
ssh 192.168.2.176 systemctl daemon-reload
ssh 192.168.2.177 systemctl daemon-reload
# 设置开机启动
ssh 192.168.2.175 systemctl enable etcd.service
ssh 192.168.2.176 systemctl enable etcd.service
ssh 192.168.2.177 systemctl enable etcd.service
# 启动etcd
ssh 192.168.2.175 systemctl  start etcd.service
ssh 192.168.2.176 systemctl  start etcd.service
ssh 192.168.2.177 systemctl  start etcd.service
# 查看启动状态
ssh 192.168.2.175 systemctl  status etcd.service
ssh 192.168.2.176 systemctl  status etcd.service
ssh 192.168.2.177 systemctl  status etcd.service
# 验证etcd 集群是否正常 任意节点
vi ~/.bashrc
export ETCDCTL_API=3
export ENDPOINTS=https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2379,https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2379,https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2379
alias ctl='/apps/etcd/bin/etcdctl   --endpoints=${ENDPOINTS}   --cacert=/apps/etcd/ssl/etcd-ca.pem --cert=/apps/etcd/ssl/etcd-client.pem --key=/apps/etcd/ssl/etcd-client-key.pem'
# 保存
source  ~/.bashrc
# 验证集群是否正常
root@k8s-master-1 conf]# ctl  endpoint status
https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2379, 6330c4573913af46, 3.4.7, 20 kB, false, false, 3, 12, 12,
https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2379, f5ee2839c4378b0, 3.4.7, 20 kB, false, false, 3, 12, 12,
https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2379, bba57102112461c, 3.4.7, 20 kB, true, false, 3, 12, 12,
[root@k8s-master-1 conf]# ctl  endpoint hashkv
https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2379, 1084519789
https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2379, 1084519789
https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2379, 1084519789
[root@k8s-master-1 conf]# ctl  endpoint health
https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2379 is healthy: successfully committed proposal: took = 22.905876ms
https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2379 is healthy: successfully committed proposal: took = 22.900899ms
https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2379 is healthy: successfully committed proposal: took = 24.118726ms
[root@k8s-master-1 conf]# ctl member list
bba57102112461c, started, k8s-master-3, https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2380, https://[fc00:bd4:efa8:1001:5054:ff:fec6:74fb]:2379, false
f5ee2839c4378b0, started, k8s-master-2, https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2380, https://[fc00:bd4:efa8:1001:5054:ff:fe47:357b]:2379, false
6330c4573913af46, started, k8s-master-1, https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2380, https://[fc00:bd4:efa8:1001:5054:ff:fe49:9888]:2379, false
# etcd 集群正常