头像 四核的基地
Article

二进制部署k8s

记录 Kubernetes 二进制部署、容器运行时切换、高可用架构设计及相关运维实践。

四核
教程
#Kubernetes

二进制部署k8s

一.k8s 1.23-将容器运行时改为containerd

1
2
3
参考链接:
	https://kubernetes.io/zh-cn/docs/tasks/administer-cluster/migrating-from-dockershim/change-runtime-containerd/
	https://kubernetes.io/zh-cn/docs/tasks/administer-cluster/migrating-from-dockershim/find-out-runtime-you-use/

1.检查现有的运行时环境

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
	1 方法一： 查看节点信息 
[root@master231 ~]# kubectl get nodes -o wide
NAME        STATUS   ROLES                  AGE     VERSION    INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
master231   Ready    control-plane,master   8d      v1.23.17   10.0.0.231    <none>        Ubuntu 22.04.4 LTS   5.15.0-136-generic   docker://20.10.24
worker232   Ready    <none>                 8d      v1.23.17   10.0.0.232    <none>        Ubuntu 22.04.4 LTS   5.15.0-136-generic   docker://20.10.24
worker233   Ready    <none>                 4d20h   v1.23.17   10.0.0.233    <none>        Ubuntu 22.04.4 LTS   5.15.0-136-generic   docker://20.10.24
[root@master231 ~]# 


	2 方法二： 检查kubelet的配置文件 
[root@master231 ~]# kubectl get nodes -o yaml | grep kubeadm.alpha.kubernetes.io/cri-socket
      kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
      kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
      kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
[root@master231 ~]#


2.驱逐worker233节点

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
	1 驱逐worker233节点
[root@master231 ~]# kubectl drain worker233 --ignore-daemonsets --delete-emptydir-data 
node/worker233 cordoned
WARNING: ignoring DaemonSet-managed Pods: kube-flannel/kube-flannel-ds-z7ndj, kube-system/kube-proxy-r68qg
evicting pod kube-system/coredns-6d8c4cb4d-zjfrb
pod/coredns-6d8c4cb4d-zjfrb evicted
node/worker233 drained
[root@master231 ~]#
[root@master231 ~]# kubectl get nodes 
NAME        STATUS                     ROLES                  AGE   VERSION
master231   Ready                      control-plane,master   20d   v1.23.17
worker232   Ready                      <none>                 20d   v1.23.17
worker233   Ready,SchedulingDisabled   <none>                 11d   v1.23.17
[root@master231 ~]# 


	2 停止kubelet进程 
[root@worker233 ~]# systemctl disable --now  kubelet
Removed /etc/systemd/system/multi-user.target.wants/kubelet.service.
[root@worker233 ~]# 
[root@worker233 ~]# systemctl disable --now docker.service 
Removed /etc/systemd/system/multi-user.target.wants/docker.service.
[root@worker233 ~]#

3.worker233更换Containerd运行时

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
	1 移除docker环境
[root@worker233 ~]# wget http://192.168.14.253/Resources/Docker/scripts/k8s-autoinstall-docker-docker-compose.tar.gz

[root@worker233 ~]# tar xf k8s-autoinstall-docker-docker-compose.tar.gz 

[root@worker233 ~]# ./install-docker.sh r  # 卸载docker环境


	2 安装Containerd
[root@worker233 ~]# wget http://192.168.14.253/Resources/Docker/Containerd/scripts/k8s-autoinstall-containerd-v1.6.36.tar.gz



[root@worker233 ~]# tar xf k8s-autoinstall-containerd-v1.6.36.tar.gz 
[root@worker233 ~]# ./install-containerd.sh i
[root@worker233 ~]# ctr version
Client:
  Version:  v1.6.36
  Revision: 88c3d9bc5b5a193f40b7c14fa996d23532d6f956
  Go version: go1.22.7

Server:
  Version:  v1.6.36
  Revision: 88c3d9bc5b5a193f40b7c14fa996d23532d6f956
  UUID: 96ed7873-56f9-4c48-a643-585ad9c51dc5
[root@worker233 ~]# 
[root@worker233 ~]# ctr ns ls  # 注意，此处安装Containerd并没有创建名称空间。
NAME LABELS 
[root@worker233 ~]# 


	3 修改kubelet的运行时环境
[root@worker233 ~]# cat /var/lib/kubelet/kubeadm-flags.env
KUBELET_KUBEADM_ARGS="--network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.6"
[root@worker233 ~]# 
[root@worker233 ~]# ll /run/containerd/containerd.sock
srw-rw---- 1 root root 0 Apr 15 11:43 /run/containerd/containerd.sock=
[root@worker233 ~]# 
[root@worker233 ~]# cat /var/lib/kubelet/kubeadm-flags.env
KUBELET_KUBEADM_ARGS="--network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.6 --container-runtime-endpoint=unix:///run/containerd/containerd.sock --container-runtime=remote"
[root@worker233 ~]# 


	4 修改Node节点的配置【此步骤可以跳过，但是建议还是做一下！】
[root@master231 ~]# kubectl get nodes 
NAME        STATUS                        ROLES                  AGE   VERSION
master231   Ready                         control-plane,master   20d   v1.23.17
worker232   Ready                         <none>                 20d   v1.23.17
worker233   NotReady,SchedulingDisabled   <none>                 11d   v1.23.17
[root@master231 ~]#
[root@master231 ~]# kubectl edit no worker233 
...
  5 apiVersion: v1
  6 kind: Node
  7 metadata:
  8   annotations:
  9     csi.volume.kubernetes.io/nodeid: '{"csi.tigera.io":"worker233"}'
 10     #kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
 11     kubeadm.alpha.kubernetes.io/cri-socket: unix:///run/containerd/containerd.sock   
 
	
	5 启动kubelet组件
[root@worker233 ~]# systemctl enable --now kubelet
Created symlink /etc/systemd/system/multi-user.target.wants/kubelet.service → /lib/systemd/system/kubelet.service.
[root@worker233 ~]# 
[root@worker233 ~]# ctr ns ls  # Kubelet默认会创建一个"k8s.io"的名称空间
NAME   LABELS 
k8s.io        
[root@worker233 ~]# 
[root@worker233 ~]# ctr -n k8s.io i ls  # 启动后Kubelet会用container拉取镜像
REF                                                                                                                               TYPE                                                      DIGEST                                                                  SIZE      PLATFORMS                                                                    LABELS                                                          
ghcr.io/flannel-io/flannel-cni-plugin:v1.7.1-flannel1                                                                             application/vnd.oci.image.index.v1+json                   sha256:cb3176a2c9eae5fa0acd7f45397e706eacb4577dac33cad89f93b775ff5611df 4.7 MiB   linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/riscv64,linux/s390x io.cri-containerd.image=managed                                 
...                               
[root@worker233 ~]# 
[root@worker233 ~]# ctr -n k8s.io c ls  # Kubelet启动的容器
CONTAINER                                                           IMAGE                                                            RUNTIME                  
096c5358694defd8ff8c9e235f53d9d1f9f44319b8b59c418365938abfd45de5    ghcr.io/flannel-io/flannel:v0.27.0                               io.containerd.runc.v2    
3cd1e87654414319011de9eb90c54b1c25d3b0c20b677ad4f2a1661686093fb2    ghcr.io/flannel-io/flannel:v0.27.0                               io.containerd.runc.v2    
5754624a6302f49b734c3794448b30d542812550b3dedb52274c3b151cf3e969    ghcr.io/flannel-io/flannel-cni-plugin:v1.7.1-flannel1            io.containerd.runc.v2    
61c2dc7459b064983078c8b9933090bb35657396978d3e2a018909bc1d77c43a    registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6    io.containerd.runc.v2    
87be262a6a4d461a56b501ed40857bc151f18c75486b80beefa0cbcaa72e790f    registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6    io.containerd.runc.v2    
f8aa72f58cde73ae0a2f6d487c3e0006c199bb67dedc3925c2ac7b4e392791b4    registry.aliyuncs.com/google_containers/kube-proxy:v1.23.17      io.containerd.runc.v2    
[root@worker233 ~]# 
[root@worker233 ~]# ctr -n k8s.io t ls  # 查看正在运行的任务
TASK                                                                PID     STATUS    
87be262a6a4d461a56b501ed40857bc151f18c75486b80beefa0cbcaa72e790f    5166    RUNNING
61c2dc7459b064983078c8b9933090bb35657396978d3e2a018909bc1d77c43a    5173    RUNNING
f8aa72f58cde73ae0a2f6d487c3e0006c199bb67dedc3925c2ac7b4e392791b4    5290    RUNNING
096c5358694defd8ff8c9e235f53d9d1f9f44319b8b59c418365938abfd45de5    5464    RUNNING
[root@worker233 ~]# 


	6.服务端再次查看底层运行时
[root@master231 ~]# kubectl get nodes -o wide
NAME        STATUS                     ROLES                  AGE   VERSION    INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
master231   Ready                      control-plane,master   20d   v1.23.17   10.0.0.231    <none>        Ubuntu 22.04.4 LTS   5.15.0-164-generic   docker://20.10.24
worker232   Ready                      <none>                 20d   v1.23.17   10.0.0.232    <none>        Ubuntu 22.04.4 LTS   5.15.0-164-generic   docker://20.10.24
worker233   Ready,SchedulingDisabled   <none>                 11d   v1.23.17   10.0.0.233    <none>        Ubuntu 22.04.4 LTS   5.15.0-164-generic   containerd://1.6.36
[root@master231 ~]#

4.取消节点不可调度

1
2
3
4
5
6
7
8
9
10
11
12
13
	1.取消不可调度
[root@master231 ~]# kubectl uncordon worker233 
node/worker233 uncordoned
[root@master231 ~]# 

	2.再次查看验证
[root@master231 ~]# kubectl get nodes -o wide
NAME        STATUS   ROLES                  AGE   VERSION    INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
master231   Ready    control-plane,master   20d   v1.23.17   10.0.0.231    <none>        Ubuntu 22.04.4 LTS   5.15.0-164-generic   docker://20.10.24
worker232   Ready    <none>                 20d   v1.23.17   10.0.0.232    <none>        Ubuntu 22.04.4 LTS   5.15.0-164-generic   docker://20.10.24
worker233   Ready    <none>                 11d   v1.23.17   10.0.0.233    <none>        Ubuntu 22.04.4 LTS   5.15.0-164-generic   containerd://1.6.36
[root@master231 ~]#

5.完成其他节点的更换操作

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
master231和worker232本质上就是在重复3-5的步骤。


	1.驱逐worker232节点
[root@master231 ~]# kubectl drain master231 --ignore-daemonsets --delete-emptydir-data 
node/master231 cordoned
WARNING: ignoring DaemonSet-managed Pods: kube-flannel/kube-flannel-ds-dlsz9, kube-system/kube-proxy-6krgk
evicting pod kube-system/coredns-6d8c4cb4d-kj8bz
evicting pod kube-system/coredns-6d8c4cb4d-dftw5
pod/coredns-6d8c4cb4d-dftw5 evicted
pod/coredns-6d8c4cb4d-kj8bz evicted
node/master231 drained
[root@master231 ~]# 


[root@master231 ~]# kubectl drain worker232 --ignore-daemonsets --delete-emptydir-data 
node/worker232 cordoned
WARNING: ignoring DaemonSet-managed Pods: kube-flannel/kube-flannel-ds-6m4pl, kube-system/kube-proxy-xjljb
evicting pod kube-system/coredns-6d8c4cb4d-bhtwz
pod/coredns-6d8c4cb4d-bhtwz evicted
node/worker232 drained
[root@master231 ~]#


温馨提示:
	master231节点应该先启动Kubelet，再去编辑，因为api-server是以静态Pod的方式运行。
	


	2.重复3-5步骤即可。

	3.验证集群是否正常工作
[root@master231 ~]# kubectl get nodes -o wide
NAME        STATUS   ROLES                  AGE     VERSION    INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
master231   Ready    control-plane,master   8d      v1.23.17   10.0.0.231    <none>        Ubuntu 22.04.4 LTS   5.15.0-136-generic   containerd://1.6.36
worker232   Ready    <none>                 8d      v1.23.17   10.0.0.232    <none>        Ubuntu 22.04.4 LTS   5.15.0-136-generic   containerd://1.6.36
worker233   Ready    <none>                 4d21h   v1.23.17   10.0.0.233    <none>        Ubuntu 22.04.4 LTS   5.15.0-136-generic   containerd://1.6.36
[root@master231 ~]# 
[root@master231 ~]# kubectl get nodes -o yaml | grep kubeadm.alpha.kubernetes.io/cri-socket
      kubeadm.alpha.kubernetes.io/cri-socket: unix:///run/containerd/containerd.sock
      kubeadm.alpha.kubernetes.io/cri-socket: unix:///run/containerd/containerd.sock
      kubeadm.alpha.kubernetes.io/cri-socket: unix:///run/containerd/containerd.sock
[root@master231 ~]#

6.检查允许时能否正常工作

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
	1.编写资源清单
[root@master231 ~]# cat > test-cni.yaml <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: xixi
spec:
  nodeName: master231
  containers:
  - image: registry.cn-hangzhou.aliyuncs.com/lkl-k8s/apps:v1
    name: c1

---

apiVersion: v1
kind: Pod
metadata:
  name: haha
spec:
  nodeName: worker232
  containers:
  - image: registry.cn-hangzhou.aliyuncs.com/lkl-k8s/apps:v2
    name: c1
    
---

apiVersion: v1
kind: Pod
metadata:
  name: hehe
spec:
  nodeName: worker233
  containers:
  - image: registry.cn-hangzhou.aliyuncs.com/lkl-k8s/apps:v2
    name: c1    
EOF


彩蛋:
	如果镜像是私有仓库地址，需要拷贝证书文件到对应的目录。
		
	参考1：
		scp harbor250.k8s.com:/usr/local/harbor/certs/docker-client/ca.crt /etc/ssl/certs/

	参考2:
		在'config.toml'配置文件中声明证书存储路径。


	2.创建资源
[root@master231 ~]# kubectl apply -f  test-cni.yaml
pod/xixi created
pod/haha created
[root@master231 ~]# 


	3.查看验证
[root@master231 ~]# kubectl get pods -o wide
NAME   READY   STATUS    RESTARTS   AGE   IP           NODE        NOMINATED NODE   READINESS GATES
haha   1/1     Running   0          6s    10.100.2.7   worker233   <none>           <none>
xixi   1/1     Running   0          6s    10.100.1.5   worker232   <none>           <none>
[root@master231 ~]#  
[root@master231 ~]# curl 10.100.2.7 
<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8"/>
    <title>yinzhengjie apps v2</title>
    <style>
       div img {
          width: 900px;
          height: 600px;
          margin: 0;
       }
    </style>
  </head>

  <body>
    <h1 style="color: red">凡人修仙传 v2 </h1>
    <div>
      <img src="2.jpg">
    <div>
  </body>

</html>
[root@master231 ~]# 
[root@master231 ~]# curl 10.100.1.5 
<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8"/>
    <title>yinzhengjie apps v1</title>
    <style>
       div img {
          width: 900px;
          height: 600px;
          margin: 0;
       }
    </style>
  </head>

  <body>
    <h1 style="color: green">凡人修仙传 v1 </h1>
    <div>
      <img src="1.jpg">
    <div>
  </body>

</html>
[root@master231 ~]#

二.k8s高可用架构设计及基础环境准备

1.环境准备

主机名/IP地址 角色分配 硬件配置
k8s-cluster241/10.0.0.241 etcd，master，worker，haproxy，keepalived 2C/4G/50G+
k8s-cluster242/10.0.0.242 etcd，master，worker，haproxy，keepalived 2C/4G/50G+
k8s-cluster243/10.0.0.243 etcd，master，worker，haproxy，keepalived 2C/4G/50G+ 
1
硬件配置如上所示。

2.k8s版本说明

1
2
2020年底kubernetes1.20宣布“弃用Dockershim”，2022年kubernetes1.24发布正式移除对Dockershim的支持，部署集群时需要单独部署，2024年1.30正式发布。
k8s1.24+版本官方默认使用Containerd运行时，因此我们不许安装docker环境

3.所有节点安装常用的软件包

1
2
apt update
apt -y install bind9-utils expect rsync jq psmisc net-tools lvm2 vim unzip rename tree

4.k8s-master241节点免密钥登录集群并同步数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
	1 设置主机名,各节点参考如下命令修改即可
[root@k8s-cluster241 ~]# hostnamectl set-hostname k8s-cluster241

	2 设置相应的主机名及hosts文件解析
[root@k8s-cluster241 ~]# cat >> /etc/hosts <<'EOF'
10.0.0.240 apiserver-lb
10.0.0.241 k8s-cluster241
10.0.0.242 k8s-cluster242
10.0.0.243 k8s-cluster243
EOF

	3 配置免密码登录其他节点
[root@k8s-cluster241 ~]# cat > password_free_login.sh <<'EOF'
#!/bin/bash
# auther: Jason Yin

# 创建密钥对
ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa -q

# 声明你服务器密码，建议所有节点的密码均一致，否则该脚本需要再次进行优化
export mypasswd=1

# 定义主机列表
k8s_host_list=(k8s-cluster242 k8s-cluster243)

# 配置免密登录，利用expect工具免交互输入
for i in ${k8s_host_list[@]};do
expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@$i
  expect {
    \"*yes/no*\" {send \"yes\r\"; exp_continue}
    \"*password*\" {send \"$mypasswd\r\"; exp_continue}
  }"
done
EOF

[root@k8s-cluster241 ~]# bash password_free_login.sh 


	4 编写同步脚本
[root@k8s-cluster241 ~]# cat > /usr/local/sbin/data_rsync.sh <<'EOF'
#!/bin/bash

# Auther: Jason Yin

if  [ $# -lt 1 ];then
   echo "Usage: $0 /path/to/file(绝对路径) [mode: m|w]"
   exit
fi 

if [ ! -e $1 ];then
    echo "[ $1 ] dir or file not find!"
    exit
fi

fullpath=`dirname $1`

basename=`basename $1`

cd $fullpath

case $2 in
    WORKER_NODE|w)
      K8S_NODE=(k8s-cluster242 k8s-cluster243)
      ;;
    MASTER_NODE|m)
      K8S_NODE=(k8s-cluster242 k8s-cluster243)
      ;;
    *)
      K8S_NODE=(k8s-cluster242 k8s-cluster243)
     ;;
esac

for host in ${K8S_NODE[@]};do
  tput setaf 2
    echo ===== rsyncing ${host}: $basename =====
    tput setaf 7
    rsync -az $basename  `whoami`@${host}:$fullpath
    if [ $? -eq 0 ];then
      echo "命令执行成功!"
    fi
done
EOF


[root@k8s-cluster241 ~]# chmod +x /usr/local/sbin/data_rsync.sh 
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# ll /usr/local/sbin/data_rsync.sh 
-rwxr-xr-x 1 root root 725 Jan 26 09:28 /usr/local/sbin/data_rsync.sh*
[root@k8s-cluster241 ~]# 


	5 同步"/etc/hosts"文件到集群
[root@k8s-cluster241 ~]# data_rsync.sh /etc/hosts

5.所有节点Linux基础环境优化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
	1 所有节点关闭NetworkManager，ufw
systemctl disable --now NetworkManager ufw 

	2 所有节点关闭swap分区，fstab注释swap
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
free -h
	
	3 手动同步时区和时间
ln -svf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

	4 所有节点配置limit
cat >> /etc/security/limits.conf <<'EOF'
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF

	5.所有节点优化sshd服务
sed -i 's@#UseDNS yes@UseDNS no@g' /etc/ssh/sshd_config
sed -i 's@^GSSAPIAuthentication yes@GSSAPIAuthentication no@g' /etc/ssh/sshd_config

	- UseDNS选项:
打开状态下，当客户端试图登录SSH服务器时，服务器端先根据客户端的IP地址进行DNS PTR反向查询出客户端的主机名，然后根据查询出的客户端主机名进行DNS正向A记录查询，验证与其原始IP地址是否一致，这是防止客户端欺骗的一种措施，但一般我们的是动态IP不会有PTR记录，打开这个选项不过是在白白浪费时间而已，不如将其关闭。

	- GSSAPIAuthentication:
当这个参数开启（ GSSAPIAuthentication  yes ）的时候，通过SSH登陆服务器时候会有些会很慢！这是由于服务器端启用了GSSAPI。登陆的时候客户端需要对服务器端的IP地址进行反解析，如果服务器的IP地址没有配置PTR记录，那么就容易在这里卡住了。


	6.Linux内核调优
cat > /etc/sysctl.d/k8s.conf <<'EOF'
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv6.conf.all.disable_ipv6 = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
sysctl --system

	7 修改终端颜色[可选]
cat <<EOF >>  ~/.bashrc 
PS1='[\[\e[34;1m\]\u@\[\e[0m\]\[\e[32;1m\]\H\[\e[0m\]\[\e[31;1m\] \W\[\e[0m\]]# '
EOF
source ~/.bashrc

6.所有节点安装ipvsadm以实现kube-proxy的负载均衡

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
	1 安装ipvsadm等相关工具
apt -y install ipvsadm ipset sysstat conntrack  

	2 所有节点创建要开机自动加载的模块配置文件
cat > /etc/modules-load.d/ipvs.conf << 'EOF'
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

	3 修改ens33网卡名称为eth0【选做，建议修改，新手跳过】
		3.1 修改配置文件
vim /etc/default/grub
...
GRUB_CMDLINE_LINUX="... net.ifnames=0 biosdevname=0"

		3.2 用grub2-mkconfig重新生成配置 
grub-mkconfig -o /boot/grub/grub.cfg 

		3.3 修改网卡配置
[root@k8s-master01 ~]# cat /etc/netplan/00-installer-config.yaml
network:
  ethernets:
    eth0:
      dhcp4: false
      addresses:
        - 10.0.0.241/24
      routes:
        - to: default
          via: 10.0.0.254
      nameservers:
        addresses:
            # 114 DNS
          - 114.114.114.114
          - 114.114.115.115
            # 阿里云DNS
          - 223.5.5.5
          - 223.6.6.6
            # 腾讯云DNS
          - 119.29.29.29
          - 119.28.28.28
            # 百度DNS
          - 180.76.76.76
            # Google DNS
          - 8.8.8.8
          - 4.4.4.4
  version: 2
[root@k8s-master01 ~]# 

	4 重启操作系统即可
reboot 


	5.验证加载的模块
lsmod | grep --color=auto -e ip_vs -e nf_conntrack
uname -r
nproc  # 查看CPU核心数量
free -h
ifconfig

温馨提示:
	Linux kernel 4.19+版本已经将之前的"nf_conntrack_ipv4"模块更名为"nf_conntrack"模块哟~

四.安装K8S相关程序

1.下载软件包

1
2
wget https://dl.k8s.io/v1.35.0/kubernetes-server-linux-amd64.tar.gz

2.解压K8S的二进制程序包到PATH环境变量路径

1
2
3
4
5
6
7
8
9
10
11
12
13
14
	1 解压软件包
[root@k8s-cluster241 ~]# tar -xf kubernetes-server-linux-amd64-v1.35.0.tar.gz  --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}


	2 查看kubelet的版本
[root@k8s-cluster241 ~]# kubelet --version
Kubernetes v1.35.0
[root@k8s-cluster241 ~]

	3.分发软件包
[root@k8s-cluster241 ~]# scp /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} k8s-cluster242:/usr/local/bin


[root@k8s-cluster241 ~]# scp /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} k8s-cluster243:/usr/local/bin

3.安装Containerd

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
	1.下载软件包
[root@k8s-cluster241 ~]# wget  http://192.168.14.253/Resources/Docker/Containerd/scripts/k8s-autoinstall-containerd-v2.2.2.tar.gz


	2.安装Containerd
[root@k8s-cluster241 ~]# tar xf k8s-autoinstall-containerd-v2.2.2.tar.gz 
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# ./install-containerd.sh i


	3.检查环境
[root@k8s-cluster241 ~]# ctr version
Client:
  Version:  v2.2.2
  Revision: 301b2dac98f15c27117da5c8af12118a041a31d9
  Go version: go1.25.7

Server:
  Version:  v2.2.2
  Revision: 301b2dac98f15c27117da5c8af12118a041a31d9
  UUID: 8b580859-4526-42e5-88a0-c1779fe40a43
[root@k8s-cluster241 ~]# 


	4.将安装脚本同步到其他节点
[root@k8s-cluster241 ~]# scp -r ./install-containerd.sh download/ k8s-cluster242:~


[root@k8s-cluster241 ~]# scp -r ./install-containerd.sh download/ k8s-cluster243:~


	5.其他节点安装Containerd
[root@k8s-cluster242 ~]# ./install-containerd.sh i

[root@k8s-cluster243 ~]# ./install-containerd.sh i

4.etcd分布式集群搭建

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
	1.下载etcd的软件包
wget https://github.com/etcd-io/etcd/releases/download/v3.6.10/etcd-v3.6.10-linux-amd64.tar.gz


	2 解压etcd的二进制程序包到PATH环境变量路径
[root@k8s-cluster241 ~]# tar -xf etcd-v3.6.10-linux-amd64.tar.gz -C /usr/local/bin etcd-v3.6.10-linux-amd64/etcd{,utl,ctl}  --strip-components=1
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# ll /usr/local/bin/etcd*
-rwxr-xr-x 1 lkl lkl 26607800 Apr  2 02:30 /usr/local/bin/etcd*
-rwxr-xr-x 1 lkl lkl 17469624 Apr  2 02:30 /usr/local/bin/etcdctl*
-rwxr-xr-x 1 lkl lkl 17617080 Apr  2 02:30 /usr/local/bin/etcdutl*
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# etcdctl version
etcdctl version: 3.6.10
API version: 3.6
[root@k8s-cluster241 ~]# 




	3 将软件包下发到所有节点
[root@k8s-cluster241 ~]# scp /usr/local/bin/etcd* k8s-cluster242:/usr/local/bin

[root@k8s-cluster241 ~]# scp /usr/local/bin/etcd* k8s-cluster243:/usr/local/bin


	4.安装cfssl证书管理工具
[root@k8s-cluster241 ~]# wget http://192.168.14.253/Resources/Prometheus/softwares/Etcd/k8s-cfssl-v1.6.5.zip

[root@k8s-cluster241 ~]# unzip k8s-cfssl-v1.6.5.zip 

[root@k8s-cluster241 ~]# rename -v "s/_1.6.5_linux_amd64//g" cfssl*

[root@k8s-cluster241 ~]# mv cfssl* /usr/local/bin/
[root@k8s-cluster241 ~]# 

[root@k8s-cluster241 ~]# chmod +x /usr/local/bin/cfssl*
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# ll /usr/local/bin/cfssl*
-rwxr-xr-x 1 root root 11890840 Jun 15  2024 /usr/local/bin/cfssl*
-rwxr-xr-x 1 root root  8413336 Jun 15  2024 /usr/local/bin/cfssl-certinfo*
-rwxr-xr-x 1 root root  6205592 Jun 15  2024 /usr/local/bin/cfssljson*
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# 

	5.创建证书存储目录
[root@k8s-cluster241 ~]# mkdir -pv /k8s/{pki,certs}/etcd
mkdir: created directory '/k8s'
mkdir: created directory '/k8s/pki'
mkdir: created directory '/k8s/pki/etcd'
mkdir: created directory '/k8s/certs'
mkdir: created directory '/k8s/certs/etcd'
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# tree /k8s/
/k8s/
├── certs
│   └── etcd
└── pki
    └── etcd

4 directories, 0 files
[root@k8s-cluster241 ~]# 





	6.生成证书的CSR文件： 证书签发请求文件，配置了一些域名，公司，单位
[root@k8s-cluster241 ~]# cd /k8s/pki/etcd/
[root@k8s-cluster241 etcd]# 
[root@k8s-cluster241 etcd]#  cat > etcd-ca-csr.json <<EOF
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ],
  "ca": {
    "expiry": "876000h"
  }
}
EOF


	7. 生成etcd CA证书和CA证书的key
[root@k8s-cluster241 etcd]# cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /k8s/certs/etcd/etcd-ca
[root@k8s-cluster241 etcd]# 
[root@k8s-cluster241 etcd]# ll /k8s/certs/etcd/etcd-ca*
-rw-r--r-- 1 root root 1050 May  4 11:09 /k8s/certs/etcd/etcd-ca.csr
-rw------- 1 root root 1679 May  4 11:09 /k8s/certs/etcd/etcd-ca-key.pem
-rw-r--r-- 1 root root 1318 May  4 11:09 /k8s/certs/etcd/etcd-ca.pem
[root@k8s-cluster241 etcd]# 


	8. 生成etcd证书的有效期为100年
[root@k8s-cluster241 etcd]# cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}
EOF


	9.生成证书的CSR文件： 证书签发请求文件，配置了一些域名，公司，单位
[root@k8s-cluster241 etcd]# cat > etcd-csr.json <<EOF
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ]
}
EOF


	10.基于自建的ectd ca证书生成etcd的证书
[root@k8s-cluster241 etcd]# cfssl gencert \
  -ca=/k8s/certs/etcd/etcd-ca.pem \
  -ca-key=/k8s/certs/etcd/etcd-ca-key.pem \
  -config=ca-config.json \
  --hostname=127.0.0.1,k8s-cluster241,k8s-cluster242,k8s-cluster243,10.0.0.241,10.0.0.242,10.0.0.243 \
  --profile=kubernetes \
  etcd-csr.json  | cfssljson -bare /k8s/certs/etcd/etcd-server


[root@k8s-cluster241 etcd]# ll /k8s/certs/etcd/etcd-server*
-rw-r--r-- 1 root root 1139 May  4 11:12 /k8s/certs/etcd/etcd-server.csr
-rw------- 1 root root 1679 May  4 11:12 /k8s/certs/etcd/etcd-server-key.pem
-rw-r--r-- 1 root root 1472 May  4 11:12 /k8s/certs/etcd/etcd-server.pem
[root@k8s-cluster241 etcd]# 



	11.将etcd证书同步到其他两个master节点
[root@k8s-cluster241 etcd]# tree /k8s/
/k8s/
├── certs
│   └── etcd
│       ├── etcd-ca.csr
│       ├── etcd-ca-key.pem
│       ├── etcd-ca.pem
│       ├── etcd-server.csr
│       ├── etcd-server-key.pem
│       └── etcd-server.pem
└── pki
    └── etcd
        ├── ca-config.json
        ├── etcd-ca-csr.json
        └── etcd-csr.json

4 directories, 9 files
[root@k8s-cluster241 etcd]# 
[root@k8s-cluster241 etcd]# 
[root@k8s-cluster241 etcd]# data_rsync.sh /k8s/
===== rsyncing k8s-cluster242: k8s =====
命令执行成功!
===== rsyncing k8s-cluster243: k8s =====
命令执行成功!
[root@k8s-cluster241 etcd]# 


	12.k8s-cluster241节点的配置文件
[root@k8s-cluster241 ~]# mkdir -pv /k8s/softwares/etcd
mkdir: created directory '/k8s/softwares'
mkdir: created directory '/k8s/softwares/etcd'
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# cat > /k8s/softwares/etcd/etcd.config.yml <<'EOF'
name: 'k8s-cluster241'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://10.0.0.241:2380'
listen-client-urls: 'https://10.0.0.241:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://10.0.0.241:2380'
advertise-client-urls: 'https://10.0.0.241:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-cluster241=https://10.0.0.241:2380,k8s-cluster242=https://10.0.0.242:2380,k8s-cluster243=https://10.0.0.243:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/k8s/certs/etcd/etcd-server.pem'
  key-file: '/k8s/certs/etcd/etcd-server-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/k8s/certs/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/k8s/certs/etcd/etcd-server.pem'
  key-file: '/k8s/certs/etcd/etcd-server-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/k8s/certs/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF



	13.k8s-cluster242节点的配置文件
[root@k8s-cluster242 ~]# mkdir -pv /k8s/softwares/etcd
mkdir: created directory '/k8s/softwares'
mkdir: created directory '/k8s/softwares/etcd'
[root@k8s-cluster242 ~]# 
[root@k8s-cluster242 ~]# cat > /k8s/softwares/etcd/etcd.config.yml <<'EOF'
name: 'k8s-cluster242'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://10.0.0.242:2380'
listen-client-urls: 'https://10.0.0.242:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://10.0.0.242:2380'
advertise-client-urls: 'https://10.0.0.242:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-cluster241=https://10.0.0.241:2380,k8s-cluster242=https://10.0.0.242:2380,k8s-cluster243=https://10.0.0.243:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/k8s/certs/etcd/etcd-server.pem'
  key-file: '/k8s/certs/etcd/etcd-server-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/k8s/certs/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/k8s/certs/etcd/etcd-server.pem'
  key-file: '/k8s/certs/etcd/etcd-server-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/k8s/certs/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF


	14.k8s-cluster243节点的配置文件
[root@k8s-cluster243 ~]# mkdir -pv /k8s/softwares/etcd
mkdir: created directory '/k8s/softwares'
mkdir: created directory '/k8s/softwares/etcd'
[root@k8s-cluster243 ~]# 
[root@k8s-cluster243 ~]# cat > /k8s/softwares/etcd/etcd.config.yml <<'EOF'
name: 'k8s-cluster243'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://10.0.0.243:2380'
listen-client-urls: 'https://10.0.0.243:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://10.0.0.243:2380'
advertise-client-urls: 'https://10.0.0.243:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-cluster241=https://10.0.0.241:2380,k8s-cluster242=https://10.0.0.242:2380,k8s-cluster243=https://10.0.0.243:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/k8s/certs/etcd/etcd-server.pem'
  key-file: '/k8s/certs/etcd/etcd-server-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/k8s/certs/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/k8s/certs/etcd/etcd-server.pem'
  key-file: '/k8s/certs/etcd/etcd-server-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/k8s/certs/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF

	15.所有节点编写etcd启动脚本
cat > /usr/lib/systemd/system/etcd.service <<'EOF'
[Unit]
Description=Jason Yin's Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/k8s/softwares/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd3.service
EOF


	16.所有节点启动etcd集群
systemctl daemon-reload && systemctl enable --now etcd
systemctl status etcd


	17.查看etcd集群状态
[root@k8s-cluster241 ~]# etcdctl --endpoints="https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379" --cacert=/k8s/certs/etcd/etcd-ca.pem --cert=/k8s/certs/etcd/etcd-server.pem --key=/k8s/certs/etcd/etcd-server-key.pem  endpoint status --write-out=table
+-------------------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
|        ENDPOINT         |        ID        | VERSION | STORAGE VERSION | DB SIZE | IN USE | PERCENTAGE NOT IN USE | QUOTA  | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | DOWNGRADE TARGET VERSION | DOWNGRADE ENABLED |
+-------------------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
| https://10.0.0.241:2379 | 566d563f3c9274ed |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |      true |      false |         2 |          9 |                  9 |        |                          |             false |
| https://10.0.0.242:2379 | b83b69ba7d246b29 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |     false |      false |         2 |          9 |                  9 |        |                          |             false |
| https://10.0.0.243:2379 |  47b70f9ecb1f200 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |     false |      false |         2 |          9 |                  9 |        |                          |             false |
+-------------------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
[root@k8s-cluster241 ~]# 


	18.验证etcd高可用集群
		18.1 停止leader节点
[root@k8s-cluster241 ~]# ss -ntl | egrep "2379|2380"
LISTEN 0      16384     10.0.0.241:2379       0.0.0.0:*          
LISTEN 0      16384     10.0.0.241:2380       0.0.0.0:*          
LISTEN 0      16384      127.0.0.1:2379       0.0.0.0:*          
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# systemctl stop etcd
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# ss -ntl | egrep "2379|2380"
[root@k8s-cluster241 ~]# 




		18.2 查看现有集群环境，发现新leader诞生
[root@k8s-cluster241 ~]# etcdctl --endpoints="https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379" --cacert=/k8s/certs/etcd/etcd-ca.pem --cert=/k8s/certs/etcd/etcd-server.pem --key=/k8s/certs/etcd/etcd-server-key.pem  endpoint status --write-out=table
{"level":"warn","ts":"2026-05-04T11:19:26.883598+0800","logger":"etcd-client","caller":"v3@v3.6.10/retry_interceptor.go:68","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00033e5a0/10.0.0.241:2379","peer":"Peer{Addr: <nil>, LocalAddr: <nil>, AuthInfo: <nil>}","method":"/etcdserverpb.Maintenance/Status","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: connection error: desc = \"transport: Error while dialing: dial tcp 10.0.0.241:2379: connect: connection refused\""}
Failed to get the status of endpoint https://10.0.0.241:2379 (context deadline exceeded)
+-------------------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
|        ENDPOINT         |        ID        | VERSION | STORAGE VERSION | DB SIZE | IN USE | PERCENTAGE NOT IN USE | QUOTA  | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | DOWNGRADE TARGET VERSION | DOWNGRADE ENABLED |
+-------------------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
| https://10.0.0.242:2379 | b83b69ba7d246b29 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |      true |      false |         3 |         10 |                 10 |        |                          |             false |
| https://10.0.0.243:2379 |  47b70f9ecb1f200 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |     false |      false |         3 |         10 |                 10 |        |                          |             false |
+-------------------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
[root@k8s-cluster241 ~]# 


		18.3 再将之前的leader起来
[root@k8s-cluster241 ~]# ss -ntl | egrep "2379|2380"
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# systemctl start etcd
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# ss -ntl | egrep "2379|2380"
LISTEN 0      16384     10.0.0.241:2379       0.0.0.0:*          
LISTEN 0      16384     10.0.0.241:2380       0.0.0.0:*          
LISTEN 0      16384      127.0.0.1:2379       0.0.0.0:*          
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# etcdctl --endpoints="https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379" --cacert=/k8s/certs/etcd/etcd-ca.pem --cert=/k8s/certs/etcd/etcd-server.pem --key=/k8s/certs/etcd/etcd-server-key.pem  endpoint status --write-out=table
+-------------------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
|        ENDPOINT         |        ID        | VERSION | STORAGE VERSION | DB SIZE | IN USE | PERCENTAGE NOT IN USE | QUOTA  | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | DOWNGRADE TARGET VERSION | DOWNGRADE ENABLED |
+-------------------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
| https://10.0.0.241:2379 | 566d563f3c9274ed |  3.6.10 |           3.6.0 |   25 kB |  25 kB |                    0% | 2.1 GB |     false |      false |         3 |         11 |                 11 |        |                          |             false |
| https://10.0.0.242:2379 | b83b69ba7d246b29 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |      true |      false |         3 |         11 |                 11 |        |                          |             false |
| https://10.0.0.243:2379 |  47b70f9ecb1f200 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |     false |      false |         3 |         11 |                 11 |        |                          |             false |
+-------------------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
[root@k8s-cluster241 ~]# 


	19.添加别名
		19.1 添加别名
[root@k8s-cluster241 ~]#  vim ~/.bashrc 
...
alias etcdctl='etcdctl --endpoints="10.0.0.241:2379,10.0.0.242:2379,10.0.0.243:2379" --cacert=/k8s/certs/etcd/etcd-ca.pem --cert=/k8s/certs/etcd/etcd-server.pem --key=/k8s/certs/etcd/etcd-server-key.pem ' 

...
[root@k8s-cluster241 ~]# source ~/.bashrc 
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# etcdctl endpoint status --write-out=table
+-----------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
|    ENDPOINT     |        ID        | VERSION | STORAGE VERSION | DB SIZE | IN USE | PERCENTAGE NOT IN USE | QUOTA  | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | DOWNGRADE TARGET VERSION | DOWNGRADE ENABLED |
+-----------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
| 10.0.0.241:2379 | 566d563f3c9274ed |  3.6.10 |           3.6.0 |   25 kB |  25 kB |                    0% | 2.1 GB |     false |      false |         3 |         11 |                 11 |        |                          |             false |
| 10.0.0.242:2379 | b83b69ba7d246b29 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |      true |      false |         3 |         11 |                 11 |        |                          |             false |
| 10.0.0.243:2379 |  47b70f9ecb1f200 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |     false |      false |         3 |         11 |                 11 |        |                          |             false |
+-----------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
[root@k8s-cluster241 ~]# 



[root@k8s-cluster241 ~]# data_rsync.sh ~/.bashrc 
===== rsyncing k8s-cluster242: .bashrc =====
命令执行成功!
===== rsyncing k8s-cluster243: .bashrc =====
命令执行成功!
[root@k8s-cluster241 ~]# 




		19.2 测试验证 【需要断开重连】
[root@k8s-cluster242 ~]# etcdctl endpoint status --write-out=table
+-----------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
|    ENDPOINT     |        ID        | VERSION | STORAGE VERSION | DB SIZE | IN USE | PERCENTAGE NOT IN USE | QUOTA  | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | DOWNGRADE TARGET VERSION | DOWNGRADE ENABLED |
+-----------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
| 10.0.0.241:2379 | 566d563f3c9274ed |  3.6.10 |           3.6.0 |   25 kB |  25 kB |                    0% | 2.1 GB |     false |      false |         3 |         11 |                 11 |        |                          |             false |
| 10.0.0.242:2379 | b83b69ba7d246b29 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |      true |      false |         3 |         11 |                 11 |        |                          |             false |
| 10.0.0.243:2379 |  47b70f9ecb1f200 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |     false |      false |         3 |         11 |                 11 |        |                          |             false |
+-----------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
[root@k8s-cluster242 ~]# 



[root@k8s-cluster243 ~]# etcdctl endpoint status -w table
+-----------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
|    ENDPOINT     |        ID        | VERSION | STORAGE VERSION | DB SIZE | IN USE | PERCENTAGE NOT IN USE | QUOTA  | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | DOWNGRADE TARGET VERSION | DOWNGRADE ENABLED |
+-----------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
| 10.0.0.241:2379 | 566d563f3c9274ed |  3.6.10 |           3.6.0 |   25 kB |  25 kB |                    0% | 2.1 GB |     false |      false |         3 |         11 |                 11 |        |                          |             false |
| 10.0.0.242:2379 | b83b69ba7d246b29 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |      true |      false |         3 |         11 |                 11 |        |                          |             false |
| 10.0.0.243:2379 |  47b70f9ecb1f200 |  3.6.10 |           3.6.0 |   20 kB |  16 kB |                   20% | 2.1 GB |     false |      false |         3 |         11 |                 11 |        |                          |             false |
+-----------------+------------------+---------+-----------------+---------+--------+-----------------------+--------+-----------+------------+-----------+------------+--------------------+--------+--------------------------+-------------------+
[root@k8s-cluster243 ~]#

五.生成k8s的master组件相关证书及kubeconfig文件

1.创建k8s证书存储目录

1
2
3
4
[root@k8s-cluster241 ~]# mkdir -pv /k8s/{certs,pki}/kubernetes/
mkdir: created directory '/k8s/certs/kubernetes/'
mkdir: created directory '/k8s/pki/kubernetes/'
[root@k8s-cluster241 ~]#

2.k8s-cluster241节点生成kubernetes自建ca证书

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
	1.生成证书的CSR文件： 证书签发请求文件，配置了一些域名，公司，单位
[root@k8s-cluster241 ~]# cd /k8s/pki/kubernetes/
[root@k8s-cluster241 kubernetes]# 
[root@k8s-cluster241 kubernetes]#  cat > k8s-ca-csr.json  <<EOF
{
"CN": "kubernetes",
"key": {
  "algo": "rsa",
  "size": 2048
},
"names": [
  {
    "C": "CN",
    "ST": "Beijing",
    "L": "Beijing",
    "O": "Kubernetes",
    "OU": "Kubernetes-manual"
  }
],
"ca": {
  "expiry": "876000h"
}
}
EOF


	2 生成kubernetes证书
[root@k8s-cluster241 kubernetes]# cfssl gencert -initca k8s-ca-csr.json | cfssljson -bare /k8s/certs/kubernetes/k8s-ca


[root@k8s-cluster241 kubernetes]# ll /k8s/certs/kubernetes/k8s-ca*
-rw-r--r-- 1 root root 1070 May  4 15:04 /k8s/certs/kubernetes/k8s-ca.csr
-rw------- 1 root root 1675 May  4 15:04 /k8s/certs/kubernetes/k8s-ca-key.pem
-rw-r--r-- 1 root root 1363 May  4 15:04 /k8s/certs/kubernetes/k8s-ca.pem
[root@k8s-cluster241 kubernetes]#

3.k8s-cluster241节点基于自建ca证书颁发apiserver相关证书

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
	1 生成k8s证书的有效期为100年
[root@k8s-cluster241 pki]# cat > k8s-ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}
EOF


	2 生成apiserver证书的CSR文件： 证书签发请求文件，配置了一些域名，公司，单位
[root@k8s-cluster241 pki]# cat > apiserver-csr.json <<EOF
{
  "CN": "kube-apiserver",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "Kubernetes",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF


	3 基于自建ca证书生成apiServer的证书文件
[root@k8s-cluster241 kubernetes]# cfssl gencert \
  -ca=/k8s/certs/kubernetes/k8s-ca.pem \
  -ca-key=/k8s/certs/kubernetes/k8s-ca-key.pem \
  -config=k8s-ca-config.json \
  --hostname=10.200.0.1,10.0.0.240,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.k8s,kubernetes.default.svc.k8s.com,10.0.0.241,10.0.0.242,10.0.0.243,apiserver-lb \
  --profile=kubernetes \
   apiserver-csr.json  | cfssljson -bare /k8s/certs/kubernetes/apiserver


[root@k8s-cluster241 kubernetes]# ll /k8s/certs/kubernetes/apiserver*
-rw-r--r-- 1 root root 1310 May  4 15:06 /k8s/certs/kubernetes/apiserver.csr
-rw------- 1 root root 1675 May  4 15:06 /k8s/certs/kubernetes/apiserver-key.pem
-rw-r--r-- 1 root root 1708 May  4 15:06 /k8s/certs/kubernetes/apiserver.pem
[root@k8s-cluster241 kubernetes]# 



温馨提示:
	"10.200.0.1"为svc网段的第一个地址，需要根据自己的场景稍作修改。
	"10.0.0.240"是负载均衡器的VIP地址。
	"kubernetes,...,kubernetes.default.svc.k8s.com"对应的是apiServer解析的A记录。
	"10.0.0.241,...,10.0.0.243"对应的是K8S集群的地址。

4.生成第三方组件与apiServer通信的聚合证书

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
聚合证书的作用就是让第三方组件(比如metrics-server等)能够拿这个证书文件和apiServer进行通信。
	
	1 生成聚合证书的用于自建ca的CSR文件
[root@k8s-cluster241 kubernetes]#  cat > front-proxy-ca-csr.json <<EOF
{
  "CN": "kubernetes",
  "key": {
     "algo": "rsa",
     "size": 2048
  }
}
EOF

	2 生成聚合证书的自建ca证书
[root@k8s-cluster241 kubernetes]# cfssl gencert -initca front-proxy-ca-csr.json | cfssljson -bare /k8s/certs/kubernetes/front-proxy-ca


[root@k8s-cluster241 kubernetes]# ll /k8s/certs/kubernetes/front-proxy-ca*
-rw-r--r-- 1 root root  891 May  4 15:07 /k8s/certs/kubernetes/front-proxy-ca.csr
-rw------- 1 root root 1675 May  4 15:07 /k8s/certs/kubernetes/front-proxy-ca-key.pem
-rw-r--r-- 1 root root 1094 May  4 15:07 /k8s/certs/kubernetes/front-proxy-ca.pem
[root@k8s-cluster241 kubernetes]# 



	3.生成聚合证书的用于客户端的CSR文件
[root@k8s-cluster241 kubernetes]# cat > front-proxy-client-csr.json <<EOF
{
  "CN": "front-proxy-client",
  "key": {
     "algo": "rsa",
     "size": 2048
  }
}
EOF


	4 基于聚合证书的自建ca证书签发聚合证书的客户端证书
[root@k8s-cluster241 kubernetes]# cfssl gencert \
  -ca=/k8s/certs/kubernetes/front-proxy-ca.pem \
  -ca-key=/k8s/certs/kubernetes/front-proxy-ca-key.pem \
  -config=k8s-ca-config.json \
  -profile=kubernetes \
  front-proxy-client-csr.json | cfssljson -bare /k8s/certs/kubernetes/front-proxy-client


[root@k8s-cluster241 kubernetes]# ll /k8s/certs/kubernetes/front-proxy-client*
-rw-r--r-- 1 root root  903 May  4 15:07 /k8s/certs/kubernetes/front-proxy-client.csr
-rw------- 1 root root 1675 May  4 15:07 /k8s/certs/kubernetes/front-proxy-client-key.pem
-rw-r--r-- 1 root root 1188 May  4 15:07 /k8s/certs/kubernetes/front-proxy-client.pem
[root@k8s-cluster241 kubernetes]#

5.生成controller-manager证书及kubeconfig文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
	1 生成controller-manager的CSR文件
[root@k8s-cluster241 kubernetes]#  cat > controller-manager-csr.json <<EOF
{
  "CN": "system:kube-controller-manager",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "system:kube-controller-manager",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF


	2 生成controller-manager证书文件
[root@k8s-cluster241 kubernetes]# cfssl gencert \
  -ca=/k8s/certs/kubernetes/k8s-ca.pem \
  -ca-key=/k8s/certs/kubernetes/k8s-ca-key.pem \
  -config=k8s-ca-config.json \
  -profile=kubernetes \
  controller-manager-csr.json | cfssljson -bare /k8s/certs/kubernetes/controller-manager


[root@k8s-cluster241 kubernetes]# ll /k8s/certs/kubernetes/controller-manager*
-rw-r--r-- 1 root root 1082 May  4 15:09 /k8s/certs/kubernetes/controller-manager.csr
-rw------- 1 root root 1679 May  4 15:09 /k8s/certs/kubernetes/controller-manager-key.pem
-rw-r--r-- 1 root root 1501 May  4 15:09 /k8s/certs/kubernetes/controller-manager.pem
[root@k8s-cluster241 kubernetes]# 



	3 创建一个kubeconfig目录
[root@k8s-cluster241 kubernetes]# mkdir -pv /k8s/certs/kubeconfig
mkdir: created directory '/k8s/certs/kubeconfig'
[root@k8s-cluster241 kubernetes]# 


	4 设置一个集群
[root@k8s-cluster241 kubernetes]#  kubectl config set-cluster lkl-k8s \
  --certificate-authority=/k8s/certs/kubernetes/k8s-ca.pem \
  --embed-certs=true \
  --server=https://10.0.0.240:8443 \
  --kubeconfig=/k8s/certs/kubeconfig/kube-controller-manager.kubeconfig
		
	5 设置一个用户项
[root@k8s-cluster241 kubernetes]#  kubectl config set-credentials system:kube-controller-manager \
  --client-certificate=/k8s/certs/kubernetes/controller-manager.pem \
  --client-key=/k8s/certs/kubernetes/controller-manager-key.pem \
  --embed-certs=true \
  --kubeconfig=/k8s/certs/kubeconfig/kube-controller-manager.kubeconfig
  
	6 设置一个上下文环境
[root@k8s-cluster241 kubernetes]#  kubectl config set-context system:kube-controller-manager@kubernetes \
  --cluster=yinzhengjie-k8s \
  --user=system:kube-controller-manager \
  --kubeconfig=/k8s/certs/kubeconfig/kube-controller-manager.kubeconfig
  
	7 使用默认的上下文
[root@k8s-cluster241 kubernetes]#  kubectl config use-context system:kube-controller-manager@kubernetes \
  --kubeconfig=/k8s/certs/kubeconfig/kube-controller-manager.kubeconfig

6.生成scheduler证书及kubeconfig文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
	1 生成scheduler的CSR文件
[root@k8s-cluster241 kubernetes]#  cat > scheduler-csr.json <<EOF
{
  "CN": "system:kube-scheduler",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "system:kube-scheduler",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF


	2 生成scheduler证书文件
[root@k8s-cluster241 kubernetes]# cfssl gencert \
  -ca=/k8s/certs/kubernetes/k8s-ca.pem \
  -ca-key=/k8s/certs/kubernetes/k8s-ca-key.pem \
  -config=k8s-ca-config.json \
  -profile=kubernetes \
  scheduler-csr.json | cfssljson -bare /k8s/certs/kubernetes/scheduler


[root@k8s-cluster241 kubernetes]# ll /k8s/certs/kubernetes/scheduler*
-rw-r--r-- 1 root root 1058 May  4 15:13 /k8s/certs/kubernetes/scheduler.csr
-rw------- 1 root root 1675 May  4 15:13 /k8s/certs/kubernetes/scheduler-key.pem
-rw-r--r-- 1 root root 1476 May  4 15:13 /k8s/certs/kubernetes/scheduler.pem
[root@k8s-cluster241 kubernetes]# 




	3.设置一个集群
[root@k8s-cluster241 kubernetes]# kubectl config set-cluster lkl-k8s \
  --certificate-authority=/k8s/certs/kubernetes/k8s-ca.pem \
  --embed-certs=true \
  --server=https://10.0.0.240:8443 \
  --kubeconfig=/k8s/certs/kubeconfig/kube-scheduler.kubeconfig
  
  
   4 设置一个用户项
[root@k8s-cluster241 kubernetes]# kubectl config set-credentials system:kube-scheduler \
  --client-certificate=/k8s/certs/kubernetes/scheduler.pem \
  --client-key=/k8s/certs/kubernetes/scheduler-key.pem \
  --embed-certs=true \
  --kubeconfig=/k8s/certs/kubeconfig/kube-scheduler.kubeconfig


	5.设置一个上下文环境
[root@k8s-cluster241 kubernetes]# kubectl config set-context system:kube-scheduler@kubernetes \
  --cluster=yinzhengjie-k8s \
  --user=system:kube-scheduler \
  --kubeconfig=/k8s/certs/kubeconfig/kube-scheduler.kubeconfig
  
	6 使用默认的上下文
[root@k8s-cluster241 kubernetes]# kubectl config use-context system:kube-scheduler@kubernetes \
  --kubeconfig=/k8s/certs/kubeconfig/kube-scheduler.kubeconfig

7.配置k8s集群管理员证书及kubeconfig文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
	1 生成管理员的CSR文件
[root@k8s-cluster241 kubernetes]# cat > admin-csr.json <<EOF
{
  "CN": "admin",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "system:masters",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF

	2.生成k8s集群管理员证书
[root@k8s-cluster241 kubernetes]# cfssl gencert \
  -ca=/k8s/certs/kubernetes/k8s-ca.pem \
  -ca-key=/k8s/certs/kubernetes/k8s-ca-key.pem \
  -config=k8s-ca-config.json \
  -profile=kubernetes \
  admin-csr.json | cfssljson -bare /k8s/certs/kubernetes/admin

[root@k8s-cluster241 kubernetes]# ll /k8s/certs/kubernetes/admin*
-rw-r--r-- 1 root root 1025 May  4 15:15 /k8s/certs/kubernetes/admin.csr
-rw------- 1 root root 1679 May  4 15:15 /k8s/certs/kubernetes/admin-key.pem
-rw-r--r-- 1 root root 1444 May  4 15:15 /k8s/certs/kubernetes/admin.pem
[root@k8s-cluster241 kubernetes]# 




	3.设置一个集群
[root@k8s-cluster241 kubernetes]# kubectl config set-cluster lkl-k8s \
  --certificate-authority=/k8s/certs/kubernetes/k8s-ca.pem \
  --embed-certs=true \
  --server=https://10.0.0.240:8443 \
  --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig
  
	4 设置一个用户项
[root@k8s-cluster241 kubernetes]# kubectl config set-credentials kube-admin \
  --client-certificate=/k8s/certs/kubernetes/admin.pem \
  --client-key=/k8s/certs/kubernetes/admin-key.pem \
  --embed-certs=true \
  --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig

	5 设置一个上下文环境
[root@k8s-cluster241 kubernetes]# kubectl config set-context kube-admin@kubernetes \
  --cluster=yinzhengjie-k8s \
  --user=kube-admin \
  --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig
  
	6.使用默认的上下文
[root@k8s-cluster241 kubernetes]# kubectl config use-context kube-admin@kubernetes \
  --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig

8 创建ServiceAccount

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
	1.OpenSSL生成RSA私钥
[root@k8s-cluster241 kubernetes]# openssl genrsa -out /k8s/certs/kubernetes/sa.key 2048
[root@k8s-cluster241 kubernetes]# 
[root@k8s-cluster241 kubernetes]# ll /k8s/certs/kubernetes/sa.key 
-rw------- 1 root root 1704 May  4 15:19 /k8s/certs/kubernetes/sa.key
[root@k8s-cluster241 kubernetes]# 


	2.从RSA私钥中提取公钥 
[root@k8s-cluster241 kubernetes]# openssl rsa -in /k8s/certs/kubernetes/sa.key -pubout -out /k8s/certs/kubernetes/sa.pub
writing RSA key
[root@k8s-cluster241 kubernetes]# 


[root@k8s-cluster241 kubernetes]# ll /k8s/certs/kubernetes/sa*
-rw------- 1 root root 1704 May  4 15:19 /k8s/certs/kubernetes/sa.key
-rw-r--r-- 1 root root  451 May  4 15:19 /k8s/certs/kubernetes/sa.pub
[root@k8s-cluster241 kubernetes]# 
[root@k8s-cluster241 kubernetes]# 


温馨提示:
	- 1.此步骤专门用于Kubernetes集群中ServiceAccount（服务账户）的密钥生成。
	- 2.api-server组件在启动时会使用'--service-account-key-file'指定公钥，使用'--service-account-signing-key-file'指定私钥。

9 k8s-master01节点K8S组件证书拷贝到其他两个master节点

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
	1 k8s-master01节点将etcd证书拷贝到其他两个master节点
[root@k8s-cluster241 kubernetes]# data_rsync.sh /k8s/certs/kubernetes/
===== rsyncing k8s-cluster242: kubernetes =====
命令执行成功!
===== rsyncing k8s-cluster243: kubernetes =====
命令执行成功!
[root@k8s-cluster241 kubernetes]# 

[root@k8s-cluster241 kubernetes]# data_rsync.sh /k8s/certs/kubeconfig/
===== rsyncing k8s-cluster242: kubeconfig =====
命令执行成功!
===== rsyncing k8s-cluster243: kubeconfig =====
命令执行成功!
[root@k8s-cluster241 kubernetes]# 



	2 其他两个节点验证文件数量是否正确
[root@k8s-cluster242 ~]# ls /k8s/certs/kubernetes  | wc -l
23
[root@k8s-cluster242 ~]# 
[root@k8s-cluster242 ~]# ls /k8s/certs/kubeconfig/  | wc -l
3
[root@k8s-cluster242 ~]# 


[root@k8s-cluster243 ~]# ls /k8s/certs/kubernetes  | wc -l
23
[root@k8s-cluster243 ~]# 
[root@k8s-cluster243 ~]# ls /k8s/certs/kubeconfig/  | wc -l
3
[root@k8s-cluster243 ~]#

六.启动K8S的master组件

1.部署ApiServer组件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
温馨提示:
  - "--advertise-address"是对应的master节点的IP地址;
  - "--service-cluster-ip-range"对应的是svc的网段
  - "--service-node-port-range"对应的是svc的NodePort端口范围;
  - "--etcd-servers"指定的是etcd集群地址

配置文件参考链接:
	https://kubernetes.io/zh-cn/docs/reference/command-line-tools-reference/kube-apiserver/
	

具体实操:
	1 创建k8s-cluster241节点的配置文件
[root@k8s-cluster241 ~]# cat > /usr/lib/systemd/system/kube-apiserver.service << 'EOF'
[Unit]
Description=Jason Yin's Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \
      --v=2  \
      --bind-address=0.0.0.0  \
      --secure-port=6443  \
      --allow_privileged=true \
      --advertise-address=10.0.0.241 \
      --service-cluster-ip-range=10.200.0.0/16  \
      --service-node-port-range=3000-50000  \
      --etcd-servers=https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379 \
      --etcd-cafile=/k8s/certs/etcd/etcd-ca.pem  \
      --etcd-certfile=/k8s/certs/etcd/etcd-server.pem  \
      --etcd-keyfile=/k8s/certs/etcd/etcd-server-key.pem  \
      --client-ca-file=/k8s/certs/kubernetes/k8s-ca.pem  \
      --tls-cert-file=/k8s/certs/kubernetes/apiserver.pem  \
      --tls-private-key-file=/k8s/certs/kubernetes/apiserver-key.pem  \
      --kubelet-client-certificate=/k8s/certs/kubernetes/apiserver.pem  \
      --kubelet-client-key=/k8s/certs/kubernetes/apiserver-key.pem  \
      --service-account-key-file=/k8s/certs/kubernetes/sa.pub  \
      --service-account-signing-key-file=/k8s/certs/kubernetes/sa.key \
      --service-account-issuer=https://kubernetes.default.svc.k8s.com \
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \
      --authorization-mode=Node,RBAC  \
      --enable-bootstrap-token-auth=true  \
      --requestheader-client-ca-file=/k8s/certs/kubernetes/front-proxy-ca.pem  \
      --proxy-client-cert-file=/k8s/certs/kubernetes/front-proxy-client.pem  \
      --proxy-client-key-file=/k8s/certs/kubernetes/front-proxy-client-key.pem  \
      --requestheader-allowed-names=aggregator  \
      --requestheader-group-headers=X-Remote-Group  \
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \
      --requestheader-username-headers=X-Remote-User

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target
EOF


	2 创建k8s-cluster242节点的配置文件
[root@k8s-cluster242 ~]# cat > /usr/lib/systemd/system/kube-apiserver.service << 'EOF'
[Unit]
Description=Jason Yin's Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \
      --v=2  \
      --bind-address=0.0.0.0  \
      --secure-port=6443  \
      --allow_privileged=true \
      --advertise-address=10.0.0.242 \
      --service-cluster-ip-range=10.200.0.0/16  \
      --service-node-port-range=3000-50000  \
      --etcd-servers=https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379 \
      --etcd-cafile=/k8s/certs/etcd/etcd-ca.pem  \
      --etcd-certfile=/k8s/certs/etcd/etcd-server.pem  \
      --etcd-keyfile=/k8s/certs/etcd/etcd-server-key.pem  \
      --client-ca-file=/k8s/certs/kubernetes/k8s-ca.pem  \
      --tls-cert-file=/k8s/certs/kubernetes/apiserver.pem  \
      --tls-private-key-file=/k8s/certs/kubernetes/apiserver-key.pem  \
      --kubelet-client-certificate=/k8s/certs/kubernetes/apiserver.pem  \
      --kubelet-client-key=/k8s/certs/kubernetes/apiserver-key.pem  \
      --service-account-key-file=/k8s/certs/kubernetes/sa.pub  \
      --service-account-signing-key-file=/k8s/certs/kubernetes/sa.key \
      --service-account-issuer=https://kubernetes.default.svc.k8s.com \
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \
      --authorization-mode=Node,RBAC  \
      --enable-bootstrap-token-auth=true  \
      --requestheader-client-ca-file=/k8s/certs/kubernetes/front-proxy-ca.pem  \
      --proxy-client-cert-file=/k8s/certs/kubernetes/front-proxy-client.pem  \
      --proxy-client-key-file=/k8s/certs/kubernetes/front-proxy-client-key.pem  \
      --requestheader-allowed-names=aggregator  \
      --requestheader-group-headers=X-Remote-Group  \
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \
      --requestheader-username-headers=X-Remote-User

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target
EOF



	3 创建k8s-cluster243节点的配置文件
[root@k8s-cluster243 ~]# cat > /usr/lib/systemd/system/kube-apiserver.service << 'EOF'
[Unit]
Description=Jason Yin's Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \
      --v=2  \
      --bind-address=0.0.0.0  \
      --secure-port=6443  \
      --allow_privileged=true \
      --advertise-address=10.0.0.243 \
      --service-cluster-ip-range=10.200.0.0/16  \
      --service-node-port-range=3000-50000  \
      --etcd-servers=https://10.0.0.241:2379,https://10.0.0.242:2379,https://10.0.0.243:2379 \
      --etcd-cafile=/k8s/certs/etcd/etcd-ca.pem  \
      --etcd-certfile=/k8s/certs/etcd/etcd-server.pem  \
      --etcd-keyfile=/k8s/certs/etcd/etcd-server-key.pem  \
      --client-ca-file=/k8s/certs/kubernetes/k8s-ca.pem  \
      --tls-cert-file=/k8s/certs/kubernetes/apiserver.pem  \
      --tls-private-key-file=/k8s/certs/kubernetes/apiserver-key.pem  \
      --kubelet-client-certificate=/k8s/certs/kubernetes/apiserver.pem  \
      --kubelet-client-key=/k8s/certs/kubernetes/apiserver-key.pem  \
      --service-account-key-file=/k8s/certs/kubernetes/sa.pub  \
      --service-account-signing-key-file=/k8s/certs/kubernetes/sa.key \
      --service-account-issuer=https://kubernetes.default.svc.k8s.com \
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \
      --authorization-mode=Node,RBAC  \
      --enable-bootstrap-token-auth=true  \
      --requestheader-client-ca-file=/k8s/certs/kubernetes/front-proxy-ca.pem  \
      --proxy-client-cert-file=/k8s/certs/kubernetes/front-proxy-client.pem  \
      --proxy-client-key-file=/k8s/certs/kubernetes/front-proxy-client-key.pem  \
      --requestheader-allowed-names=aggregator  \
      --requestheader-group-headers=X-Remote-Group  \
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \
      --requestheader-username-headers=X-Remote-User

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target
EOF


	4 所有节点启动服务
systemctl daemon-reload && systemctl enable --now kube-apiserver
systemctl status kube-apiserver
ss -ntl | grep 6443


2.部署ControlerManager组件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
温馨提示:

- "--cluster-cidr"是Pod的网段地址，我们可以自行修改。

配置文件参考链接:
	https://kubernetes.io/zh-cn/docs/reference/command-line-tools-reference/kube-controller-manager/
	 

	1.所有节点的controller-manager组件配置文件相同: (前提是证书文件存放的位置也要相同哟!)
cat > /usr/lib/systemd/system/kube-controller-manager.service << 'EOF'
[Unit]
Description=Jason Yin's Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
      --v=2 \
      --root-ca-file=/k8s/certs/kubernetes/k8s-ca.pem \
      --cluster-signing-cert-file=/k8s/certs/kubernetes/k8s-ca.pem \
      --cluster-signing-key-file=/k8s/certs/kubernetes/k8s-ca-key.pem \
      --service-account-private-key-file=/k8s/certs/kubernetes/sa.key \
      --kubeconfig=/k8s/certs/kubeconfig/kube-controller-manager.kubeconfig \
      --leader-elect=true \
      --use-service-account-credentials=true \
      --node-monitor-grace-period=40s \
      --node-monitor-period=5s \
      --controllers=*,bootstrapsigner,tokencleaner \
      --allocate-node-cidrs=true \
      --cluster-cidr=10.100.0.0/16 \
      --requestheader-client-ca-file=/k8s/certs/kubernetes/front-proxy-ca.pem \
      --node-cidr-mask-size=24
      
Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target
EOF


	2.启动controller-manager服务
systemctl daemon-reload
systemctl enable --now kube-controller-manager
systemctl  status kube-controller-manager
ss -ntl | grep 10257

3.部署Scheduler组件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
	1 所有节点创建配置文件
配置文件参考链接:
https://kubernetes.io/zh-cn/docs/reference/command-line-tools-reference/kube-scheduler/

所有节点的controller-manager组件配置文件相同: (前提是证书文件存放的位置也要相同哟!)
cat > /usr/lib/systemd/system/kube-scheduler.service <<'EOF'
[Unit]
Description=Jason Yin's Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-scheduler \
      --v=2 \
      --leader-elect=true \
      --kubeconfig=/k8s/certs/kubeconfig/kube-scheduler.kubeconfig

Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target
EOF


	2 启动scheduler服务
systemctl daemon-reload
systemctl enable --now kube-scheduler
systemctl  status kube-scheduler
ss -ntl | grep 10259

4.测试组件是否正常

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
	1.临时修改api-server的地址和端口
[root@k8s-cluster241 ~]# grep server  /k8s/certs/kubeconfig/kube-admin.kubeconfig
    # server: https://10.0.0.240:8443
    server: https://10.0.0.241:6443
[root@k8s-cluster241 ~]# 


	2.临时测试集群是否可用
[root@k8s-cluster241 ~]# kubectl get cs --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE   ERROR
scheduler            Healthy   ok        
controller-manager   Healthy   ok        
etcd-0               Healthy   ok        
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# kubectl get nodes --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig
No resources found
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# kubectl get ns  --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig 
NAME              STATUS   AGE
default           Active   3m30s
kube-node-lease   Active   3m30s
kube-public       Active   3m30s
kube-system       Active   3m30s
[root@k8s-cluster241 ~]# 



	3.将配置还原
[root@k8s-cluster241 ~]# grep server  /k8s/certs/kubeconfig/kube-admin.kubeconfig
    server: https://10.0.0.240:8443
    # server: https://10.0.0.241:6443
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# kubectl get cs --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig
E0126 12:12:16.875493    4103 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://10.0.0.240:8443/api?timeout=32s\": dial tcp 10.0.0.240:8443: connect: no route to host"
E0126 12:12:19.943112    4103 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://10.0.0.240:8443/api?timeout=32s\": dial tcp 10.0.0.240:8443: connect: no route to host"
E0126 12:12:23.015217    4103 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://10.0.0.240:8443/api?timeout=32s\": dial tcp 10.0.0.240:8443: connect: no route to host"
E0126 12:12:26.086947    4103 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://10.0.0.240:8443/api?timeout=32s\": dial tcp 10.0.0.240:8443: connect: no route to host"
E0126 12:12:29.158978    4103 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://10.0.0.240:8443/api?timeout=32s\": dial tcp 10.0.0.240:8443: connect: no route to host"
Unable to connect to the server: dial tcp 10.0.0.240:8443: connect: no route to host
[root@k8s-cluster241 ~]#

七.高可用组件haproxy+keepalived安装及验证

1.所有master【k8s-cluster24[1-3]】节点安装高可用组件

1
2
3
4
5
6
7
8
温馨提示:
    - 对于高可用组件，其实我们也可以单独找两台虚拟机来部署，但我为了节省2台机器，就直接在master节点复用了。
    - 如果在云上安装K8S则无安装高可用组件了，毕竟公有云大部分都是不支持keepalived的，可以直接使用云产品，比如阿里的"SLB"，腾讯的"ELB"等SAAS产品;
    - 推荐使用ELB，SLB有回环的问题，也就是SLB代理的服务器不能反向访问SLB，但是腾讯云修复了这个问题;

具体实操:
	apt-get -y install keepalived haproxy

2.所有master节点配置haproxy

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
温馨提示:
	- haproxy的负载均衡器监听地址我配置是8443，可以修改为其他端口，haproxy会用来反向代理各个master组件的地址;
	- 如果真的修改一定注意上面的证书配置的kubeconfig文件，也要一起修改，否则就会出现链接集群失败的问题;
	
具体实操:
	1 备份配置文件
cp /etc/haproxy/haproxy.cfg{,`date +%F`}

	2 所有节点的配置文件内容相同
cat > /etc/haproxy/haproxy.cfg <<'EOF'
global
  maxconn  2000
  ulimit-n  16384
  log  127.0.0.1 local0 err
  stats timeout 30s

defaults
  log global
  mode  http
  option  httplog
  timeout connect 5000
  timeout client  50000
  timeout server  50000
  timeout http-request 15s
  timeout http-keep-alive 15s

frontend monitor-haproxy
  bind *:9999
  mode http
  option httplog
  monitor-uri /ruok

frontend lkl-k8s-frontend
  bind 0.0.0.0:8443
  bind 127.0.0.1:8443
  mode tcp
  option tcplog
  tcp-request inspect-delay 5s
  default_backend lkl-k8s-backend

backend lkl-k8s-backend
  mode tcp
  option tcplog
  option tcp-check
  balance roundrobin
  default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
  server k8s-cluster241   10.0.0.241:6443  check
  server k8s-cluster242   10.0.0.242:6443  check
  server k8s-cluster243   10.0.0.243:6443  check
EOF

3.所有master节点配置keepalived

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
温馨提示:
	- 注意"interface"字段为你的物理网卡的名称，如果你的网卡是ens33，请将"eth0"修改为"ens33";
	- 注意"mcast_src_ip"各master节点的配置均不相同，修改根据实际环境进行修改哟;
	- 注意"virtual_ipaddress"指定的是负载均衡器的VIP地址，这个地址也要和kubeconfig文件的Apiserver地址要一致;
	- 注意"script"字段的脚本用于检测后端的apiServer是否健康;
	- 注意"router_id"字段为节点ip，master每个节点配置自己的IP
	
具体实操:
	1."k8s-cluster241"节点创建配置文件
[root@k8s-cluster241 ~]# ifconfig 
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.241  netmask 255.255.255.0  broadcast 10.0.0.255
        ether 00:0c:29:32:73:ac  txqueuelen 1000  (Ethernet)
        RX packets 324292  bytes 234183010 (223.3 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 242256  bytes 31242156 (29.7 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

...

[root@k8s-cluster241 ~]# cat > /etc/keepalived/keepalived.conf <<'EOF'
! Configuration File for keepalived
global_defs {
   router_id 10.0.0.241
}
vrrp_script check_haproxy {
    script "/etc/keepalived/check_port.sh 8443"
    interval 2
    weight -20
}
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 10.0.0.241
    nopreempt
    authentication {
        auth_type PASS
        auth_pass yinzhengjie_k8s
    }
    track_script {
         check_haproxy
    }
    virtual_ipaddress {
        10.0.0.240
    }
}
EOF


	2."k8s-cluster242"节点创建配置文件
[root@k8s-cluster242 ~]# ifconfig 
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.242  netmask 255.255.255.0  broadcast 10.0.0.255
        ether 00:0c:29:cf:ad:0a  txqueuelen 1000  (Ethernet)
        RX packets 256743  bytes 42628265 (40.6 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 252589  bytes 34277384 (32.6 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

...

[root@k8s-cluster242 ~]# cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
   router_id 10.0.0.242
}
vrrp_script check_haproxy {
    script "/etc/keepalived/check_port.sh 8443"
    interval 2
    weight -20
}
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 10.0.0.242
    nopreempt
    authentication {
        auth_type PASS
        auth_pass yinzhengjie_k8s
    }
    track_script {
         check_haproxy
    }
    virtual_ipaddress {
        10.0.0.240
    }
}
EOF


	3."k8s-cluster243"节点创建配置文件
[root@k8s-cluster243 ~]# ifconfig 
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.243  netmask 255.255.255.0  broadcast 10.0.0.255
        ether 00:0c:29:5f:f7:4f  txqueuelen 1000  (Ethernet)
        RX packets 178577  bytes 34808750 (33.1 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 171025  bytes 26471309 (25.2 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

...

[root@k8s-cluster243 ~]# cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
   router_id 10.0.0.243
}
vrrp_script check_haproxy {
    script "/etc/keepalived/check_port.sh 8443"
    interval 2
    weight -20
}
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 10.0.0.243
    nopreempt
    authentication {
        auth_type PASS
        auth_pass yinzhengjie_k8s
    }
    track_script {
         check_haproxy
    }
    virtual_ipaddress {
        10.0.0.240
    }
}
EOF


	4.所有keepalived节点均需要创建健康检查脚本
cat > /etc/keepalived/check_port.sh <<'EOF'
#!/bin/bash
CHK_PORT=$1
if [ -n "$CHK_PORT" ];then
    PORT_PROCESS=`ss -lt|grep $CHK_PORT|wc -l`
    if [ $PORT_PROCESS -eq 0 ];then
        echo "Port $CHK_PORT Is Not Used,End."
        systemctl stop keepalived
    fi
else
    echo "Check Port Cant Be Empty!"
fi
EOF
chmod +x /etc/keepalived/check_port.sh

4.验证haproxy服务并验证

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
	1 所有节点启动haproxy服务
systemctl enable --now haproxy 
systemctl restart haproxy 
systemctl status haproxy 
ss -ntl | grep 8443

	2 所有节点启动keepalived 
systemctl start keepalived

	3 基于telnet验证haporxy是否正常
[root@k8s-cluster243 ~]# telnet 10.0.0.240 8443
Trying 10.0.0.240...
Connected to 10.0.0.240.
Escape character is '^]'.

	4 基于webUI进行验证
[root@k8s-cluster242 ~]# curl http://10.0.0.240:9999/ruok
<html><body><h1>200 OK</h1>
Service ready.
</body></html>
[root@k8s-cluster242 ~]# 


	5.后期优化【目的是启动keepalived之前，应该先启动haproxy】
[root@k8s-cluster241 ~]# cat  /lib/systemd/system/keepalived.service 
[Unit]
Description=Keepalive Daemon (LVS and VRRP)
After=network-online.target,haproxy.service  # 目的是为了先启动haproxy组件，再启动keepalived组件。
Wants=network-online.target
# Only start if there is a configuration file
ConditionFileNotEmpty=/etc/keepalived/keepalived.conf

[Service]
Type=notify
# Read configuration variable file if it is present
EnvironmentFile=-/etc/default/keepalived
ExecStart=/usr/sbin/keepalived --dont-fork $DAEMON_ARGS
ExecReload=/bin/kill -HUP $MAINPID

[Install]
WantedBy=multi-user.target
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# data_rsync.sh   /lib/systemd/system/keepalived.service 
===== rsyncing k8s-cluster242: keepalived.service =====
命令执行成功!
===== rsyncing k8s-cluster243: keepalived.service =====
命令执行成功!
[root@k8s-cluster241 ~]#

5.kubectl测试验证

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
	1.各个master节点测试
[root@k8s-cluster241 ~]# kubectl get cs --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE   ERROR
controller-manager   Healthy   ok        
scheduler            Healthy   ok        
etcd-0               Healthy   ok        
[root@k8s-cluster241 ~]# 
	
	
[root@k8s-cluster242 ~]# kubectl get cs --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE   ERROR
controller-manager   Healthy   ok        
scheduler            Healthy   ok        
etcd-0               Healthy   ok        
[root@k8s-cluster242 ~]# 


[root@k8s-cluster243 ~]# kubectl get cs --kubeconfig=/k8s/certs/kubeconfig/kube-admin.kubeconfig
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE   ERROR
scheduler            Healthy   ok        
controller-manager   Healthy   ok        
etcd-0               Healthy   ok        
[root@k8s-cluster243 ~]# 



	2.准备默认的文件
[root@k8s-cluster241 ~]# mkdir -p  ~/.kube
[root@k8s-cluster241 ~]# cp /k8s/certs/kubeconfig/kube-admin.kubeconfig ~/.kube/config
[root@k8s-cluster241 ~]# kubectl get cs 
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE   ERROR
controller-manager   Healthy   ok        
scheduler            Healthy   ok        
etcd-0               Healthy   ok        
[root@k8s-cluster241 ~]# 


[root@k8s-cluster242 ~]# mkdir -p  ~/.kube
[root@k8s-cluster242 ~]# cp /k8s/certs/kubeconfig/kube-admin.kubeconfig ~/.kube/config
[root@k8s-cluster242 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE   ERROR
controller-manager   Healthy   ok        
scheduler            Healthy   ok        
etcd-0               Healthy   ok        
[root@k8s-cluster242 ~]# 


[root@k8s-cluster243 ~]# mkdir -p  ~/.kube
[root@k8s-cluster243 ~]# cp /k8s/certs/kubeconfig/kube-admin.kubeconfig ~/.kube/config
[root@k8s-cluster243 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE   ERROR
controller-manager   Healthy   ok        
scheduler            Healthy   ok        
etcd-0               Healthy   ok        
[root@k8s-cluster243 ~]# 


	3.各节点配置自动补全功能
kubectl completion bash > ~/.kube/completion.bash.inc
echo source '$HOME/.kube/completion.bash.inc' >> ~/.bashrc 
source ~/.bashrc
kubectl  # 连续按2次tab键，观察是否自动补全，如上图所示。

八.启动K8S的worker组件

1.创建Bootstrapping自动颁发kubelet证书配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
温馨提示:
	- "--server"只想的是负载均衡器的IP地址，由负载均衡器对master节点进行反向代理。
	- "--token"也可以自定义，但也要同时修改"bootstrap"的Secret的"token-id"和"token-secret"对应值;


	1 设置集群
[root@k8s-cluster241 ~]# kubectl config set-cluster lkl-k8s \
  --certificate-authority=/k8s/certs/kubernetes/k8s-ca.pem \
  --embed-certs=true \
  --server=https://10.0.0.240:8443 \
  --kubeconfig=/k8s/certs/kubeconfig/bootstrap-kubelet.kubeconfig

	2 创建用户
[root@k8s-cluster241 ~]# kubectl config set-credentials tls-bootstrap-token-user  \
  --token=yindao.jasonyinzhengjie \
  --kubeconfig=/k8s/certs/kubeconfig/bootstrap-kubelet.kubeconfig
  
	3 将集群和用户进行绑定
[root@k8s-cluster241 ~]# kubectl config set-context tls-bootstrap-token-user@kubernetes \
  --cluster=yinzhengjie-k8s \
  --user=tls-bootstrap-token-user \
  --kubeconfig=/k8s/certs/kubeconfig/bootstrap-kubelet.kubeconfig
  
  	4.配置默认的上下文
[root@k8s-cluster241 ~]# kubectl config use-context tls-bootstrap-token-user@kubernetes \
  --kubeconfig=/k8s/certs/kubeconfig/bootstrap-kubelet.kubeconfig


	5 创建配bootstrap-secret文件用于授权
[root@k8s-cluster241 ~]# cat > bootstrap-secret.yaml <<EOF
apiVersion: v1
kind: Secret
metadata:
  name: bootstrap-token-yindao
  namespace: kube-system
type: bootstrap.kubernetes.io/token
stringData:
  description: "The default bootstrap token generated by 'kubelet '."
  token-id: yindao
  token-secret: jasonyinzhengjie
  usage-bootstrap-authentication: "true"
  usage-bootstrap-signing: "true"
  auth-extra-groups:  system:bootstrappers:default-node-token,system:bootstrappers:worker,system:bootstrappers:ingress

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kubelet-bootstrap
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:node-bootstrapper
subjects:

- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:bootstrappers:default-node-token

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: node-autoapprove-bootstrap
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
subjects:

- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:bootstrappers:default-node-token

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: node-autoapprove-certificate-rotation
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
subjects:

- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:nodes

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:kube-apiserver-to-kubelet
rules:
  - apiGroups:
    - ""
    resources:
    - nodes/proxy
    - nodes/stats
    - nodes/log
    - nodes/spec
    - nodes/metrics
    verbs:
    - "*"

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:kube-apiserver
  namespace: ""
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kube-apiserver-to-kubelet
subjects:

  - apiGroup: rbac.authorization.k8s.io
    kind: User
    name: kube-apiserver
EOF


	6.应用bootstrap-secret配置文件
[root@k8s-cluster241 ~]# kubectl apply -f bootstrap-secret.yaml 
secret/bootstrap-token-yindao created
clusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created
clusterrolebinding.rbac.authorization.k8s.io/node-autoapprove-bootstrap created
clusterrolebinding.rbac.authorization.k8s.io/node-autoapprove-certificate-rotation created
clusterrole.rbac.authorization.k8s.io/system:kube-apiserver-to-kubelet created
clusterrolebinding.rbac.authorization.k8s.io/system:kube-apiserver created
[root@k8s-cluster241 ~]#

2.部署worker节点之kubelet启动实战

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
	1 同步kubelet的kubeconfig认证文件
[root@k8s-cluster241 ~]# data_rsync.sh /k8s/certs/kubeconfig/bootstrap-kubelet.kubeconfig 
===== rsyncing k8s-cluster242: bootstrap-kubelet.kubeconfig =====
命令执行成功!
===== rsyncing k8s-cluster243: bootstrap-kubelet.kubeconfig =====
命令执行成功!
[root@k8s-cluster241 ~]# 


	2.所有节点创建工作目录
mkdir -p /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/


	3.所有节点创建kubelet的配置文件
	
温馨提示:
	- 在"10-kubelet.con"文件中使用"--kubeconfig"指定的"kubelet.kubeconfig"文件并不存在，这个证书文件后期会自动生成;
	- 对于"clusterDNS"是NDS地址，我们可以自定义，比如"10.200.0.254";
	- “clusterDomain”对应的是域名信息，要和我们设计的集群保持一致，比如"lkl.com";
	- "10-kubelet.conf"文件中的"ExecStart="需要写2次，否则可能无法启动kubelet;
	
cat > /etc/kubernetes/kubelet-conf.yml <<'EOF'
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /k8s/certs/kubernetes/k8s-ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- 10.200.0.254
clusterDomain: k8s.com
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
  imagefs.available: 15%
  memory.available: 100Mi
  nodefs.available: 10%
  nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
EOF


	4.所有节点配置kubelet service
cat > /usr/lib/systemd/system/kubelet.service <<'EOF'
[Unit]
Description=JasonYin's Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=containerd.service
Requires=containerd.service

[Service]
ExecStart=/usr/local/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF


	5 所有节点配置kubelet service的配置文件
cat > /etc/systemd/system/kubelet.service.d/10-kubelet.conf <<'EOF'
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/k8s/certs/kubeconfig/bootstrap-kubelet.kubeconfig --kubeconfig=/k8s/certs/kubeconfig/kubelet.kubeconfig"
Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml"
Environment="KUBELET_SYSTEM_ARGS=--container-runtime-endpoint=unix:///run/containerd/containerd.sock"
Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node='' "
ExecStart=
ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_SYSTEM_ARGS $KUBELET_EXTRA_ARGS
EOF


	6.启动所有节点kubelet
systemctl daemon-reload
systemctl enable --now kubelet
systemctl status kubelet


	8.在所有master节点上查看nodes信息。
[root@k8s-cluster243 ~]# kubectl  get nodes -o wide
NAME             STATUS     ROLES    AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
k8s-cluster241   NotReady   <none>   12s   v1.35.0   10.0.0.241    <none>        Ubuntu 22.04.4 LTS   5.15.0-119-generic   containerd://2.2.2
k8s-cluster242   NotReady   <none>   10s   v1.35.0   10.0.0.242    <none>        Ubuntu 22.04.4 LTS   5.15.0-119-generic   containerd://2.2.2
k8s-cluster243   NotReady   <none>   7s    v1.35.0   10.0.0.243    <none>        Ubuntu 22.04.4 LTS   5.15.0-119-generic   containerd://2.2.2
[root@k8s-cluster243 ~]# 
[root@k8s-cluster243 ~]# 



	7 可以查看到有相应的csr用户客户端的证书请求
[root@k8s-cluster243 ~]# kubectl  get csr
NAME        AGE   SIGNERNAME                                    REQUESTOR                 REQUESTEDDURATION   CONDITION
csr-97tkj   25s   kubernetes.io/kube-apiserver-client-kubelet   system:bootstrap:yindao   <none>              Approved,Issued
csr-btd4j   27s   kubernetes.io/kube-apiserver-client-kubelet   system:bootstrap:yindao   <none>              Approved,Issued
csr-p229r   22s   kubernetes.io/kube-apiserver-client-kubelet   system:bootstrap:yindao   <none>              Approved,Issued
[root@k8s-cluster243 ~]#


3.部署worker节点之kube-proxy服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
	1.生成kube-proxy的csr文件
[root@k8s-cluster241 ~]# cd /k8s/pki/kubernetes/
[root@k8s-cluster241 kubernetes]# 
[root@k8s-cluster241 kubernetes]# cat > kube-proxy-csr.json  <<EOF
{
"CN": "system:kube-proxy",
"key": {
  "algo": "rsa",
  "size": 2048
},
"names": [
  {
    "C": "CN",
    "ST": "Beijing",
    "L": "Beijing",
    "O": "system:kube-proxy",
    "OU": "Kubernetes-manual"
  }
]
}
EOF


	2.创建kube-proxy需要的证书文件
[root@k8s-cluster241 kubernetes]# cfssl gencert \
-ca=/k8s/certs/kubernetes/k8s-ca.pem \
-ca-key=/k8s/certs/kubernetes/k8s-ca-key.pem \
-config=k8s-ca-config.json \
-profile=kubernetes \
kube-proxy-csr.json | cfssljson -bare /k8s/certs/kubernetes/kube-proxy



[root@k8s-cluster241 kubernetes]# ll /k8s/certs/kubernetes/kube-proxy*
-rw-r--r-- 1 root root 1045 May  4 16:39 /k8s/certs/kubernetes/kube-proxy.csr
-rw------- 1 root root 1679 May  4 16:39 /k8s/certs/kubernetes/kube-proxy-key.pem
-rw-r--r-- 1 root root 1464 May  4 16:39 /k8s/certs/kubernetes/kube-proxy.pem
[root@k8s-cluster241 kubernetes]# 



	3.设置集群
[root@k8s-cluster241 kubernetes]#  kubectl config set-cluster lkl-k8s \
  --certificate-authority=/k8s/certs/kubernetes/k8s-ca.pem \
  --embed-certs=true \
  --server=https://10.0.0.240:8443 \
  --kubeconfig=/k8s/certs/kubeconfig/kube-proxy.kubeconfig
  
  
	4.设置一个用户项
[root@k8s-cluster241 kubernetes]# kubectl config set-credentials system:kube-proxy \
  --client-certificate=/k8s/certs/kubernetes/kube-proxy.pem \
  --client-key=/k8s/certs/kubernetes/kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=/k8s/certs/kubeconfig/kube-proxy.kubeconfig

	5.设置一个上下文环境
[root@k8s-cluster241 kubernetes]# kubectl config set-context kube-proxy@kubernetes \
  --cluster=yinzhengjie-k8s \
  --user=system:kube-proxy \
  --kubeconfig=/k8s/certs/kubeconfig/kube-proxy.kubeconfig

	6.使用默认的上下文
[root@k8s-cluster241 kubernetes]# kubectl config use-context kube-proxy@kubernetes \
  --kubeconfig=/k8s/certs/kubeconfig/kube-proxy.kubeconfig
  
	7.将kube-proxy的systemd Service文件发送到其他节点
[root@k8s-cluster241 kubernetes]# data_rsync.sh /k8s/certs/kubeconfig/kube-proxy.kubeconfig 
===== rsyncing k8s-cluster242: kube-proxy.kubeconfig =====
命令执行成功!
===== rsyncing k8s-cluster243: kube-proxy.kubeconfig =====
命令执行成功!
[root@k8s-cluster241 kubernetes]# 


	8.所有节点创建kube-proxy.conf配置文件
cat > /etc/kubernetes/kube-proxy.yml << EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
metricsBindAddress: 127.0.0.1:10249
clientConnection:
  acceptConnection: ""
  burst: 10
  contentType: application/vnd.kubernetes.protobuf
  kubeconfig: /k8s/certs/kubeconfig/kube-proxy.kubeconfig
  qps: 5
clusterCIDR: 10.100.0.0/16
configSyncPeriod: 15m0s
conntrack:
  max: null
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: 14
  minSyncPeriod: 0s
ipvs:
  masqueradeAll: true
  minSyncPeriod: 5s
  scheduler: "rr"
  syncPeriod: 30s
mode: "ipvs"
nodeProtAddress: null
oomScoreAdj: -999
portRange: ""
udpIdelTimeout: 250ms
EOF


	9.所有节点使用systemd管理kube-proxy
cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Jason Yin's Kubernetes Proxy
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-proxy \
  --config=/etc/kubernetes/kube-proxy.yml \
  --v=2 
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF


	10.所有节点启动kube-proxy
systemctl daemon-reload && systemctl enable --now kube-proxy
systemctl status kube-proxy
ss -ntl |grep 10249

九.部署calico组件并验证测试

1.calico和k8s版本兼容性说明

1
2
3
参考链接:
	https://docs.tigera.io/calico/latest/getting-started/kubernetes/requirements

2.部署calico

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
温馨提示:
	做本实验之前，建议你先拍个快照，因为我们k8s版本是最新版本，calico目前还不支持最新版本。
	经测试，尽管能部署成功，但是会有部分pod会自动重启，稳定性待进一步调优测试。
	当然，我们也可以考虑使用Flannel网络插件来解决此问题。
	
	

	1.下载Tigera Operator的资源清单
[root@k8s-cluster241 ~]# wget https://raw.githubusercontent.com/projectcalico/calico/v3.32.0/manifests/v1_crd_projectcalico_org.yaml


[root@k8s-cluster241 ~]#  wget https://raw.githubusercontent.com/projectcalico/calico/v3.32.0/manifests/tigera-operator.yaml


SVIP:
wget http://192.168.14.253/Resources/Kubernetes/K8S%20Cluster/CNI/calico/calico-v3.32.0/v1_crd_projectcalico_org.yaml

wget http://192.168.14.253/Resources/Kubernetes/K8S%20Cluster/CNI/calico/calico-v3.32.0/tigera-operator.yaml

	
	
 	2.安装Tigera Operator组件
[root@k8s-cluster241 ~]# kubectl  create  -f v1_crd_projectcalico_org.yaml 
	
[root@k8s-cluster241 ~]# kubectl create -f tigera-operator.yaml 
 


	3.下载自定义资源清单模板
[root@k8s-cluster241 ~]# wget https://raw.githubusercontent.com/projectcalico/calico/v3.32.0/manifests/custom-resources.yaml


SVIP:
[root@k8s-cluster241 ~]# wget http://192.168.14.253/Resources/Kubernetes/K8S%20Cluster/CNI/calico/calico-v3.32.0/custom-resources.yaml
	
	
	
	4.通过创建必要的自定义资源来安装Calico
[root@k8s-cluster241 ~]# grep cidr custom-resources.yaml
        cidr: 192.168.0.0/16
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# sed -i '/cidr/s#192.168#10.100#' custom-resources.yaml 
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# grep cidr custom-resources.yaml
        cidr: 10.100.0.0/16
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# kubectl create -f custom-resources.yaml 
installation.operator.tigera.io/default created
apiserver.operator.tigera.io/default created
goldmane.operator.tigera.io/default created
whisker.operator.tigera.io/default created
[root@k8s-cluster241 ~]# 

	5.检查pod是否正常运行
[root@k8s-cluster241 ~]# kubectl  get pods -A
NAMESPACE         NAME                                       READY   STATUS    RESTARTS   AGE
calico-system     calico-apiserver-7798bfb5dd-8jfmn          1/1     Running   0          8m52s
calico-system     calico-apiserver-7798bfb5dd-d82r7          1/1     Running   0          8m52s
calico-system     calico-kube-controllers-669c5c99bf-kdwsg   1/1     Running   0          8m51s
calico-system     calico-node-dzcmv                          1/1     Running   0          8m51s
calico-system     calico-node-rzh6t                          1/1     Running   0          8m51s
calico-system     calico-node-wnrsg                          1/1     Running   0          8m51s
calico-system     calico-typha-697778bd48-l7hgz              1/1     Running   0          8m51s
calico-system     calico-typha-697778bd48-lpstp              1/1     Running   0          8m42s
calico-system     csi-node-driver-257hw                      2/2     Running   0          8m51s
calico-system     csi-node-driver-5tlpl                      2/2     Running   0          8m51s
calico-system     csi-node-driver-8498m                      2/2     Running   0          8m51s
calico-system     goldmane-5656bd5586-mp2x8                  1/1     Running   0          8m51s
calico-system     whisker-7bc7f99bd5-4vb8z                   2/2     Running   0          7m17s
tigera-operator   tigera-operator-665f4f4489-b89bc           1/1     Running   0          13m
[root@k8s-cluster241 ~]# 


	6.检查k8S集群节点
[root@k8s-cluster241 ~]# kubectl  get nodes -o wide
NAME             STATUS   ROLES    AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
k8s-cluster241   Ready    <none>   32m   v1.35.0   10.0.0.241    <none>        Ubuntu 22.04.4 LTS   5.15.0-119-generic   containerd://2.2.2
k8s-cluster242   Ready    <none>   32m   v1.35.0   10.0.0.242    <none>        Ubuntu 22.04.4 LTS   5.15.0-119-generic   containerd://2.2.2
k8s-cluster243   Ready    <none>   32m   v1.35.0   10.0.0.243    <none>        Ubuntu 22.04.4 LTS   5.15.0-119-generic   containerd://2.2.2
[root@k8s-cluster241 ~]#

3.测试calico的可用性

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
温馨提示:
	如果往网络不通，可以尝试删除calico-node相关的pod测试观察。
		kubectl -n calico-system delete pods -l  app.kubernetes.io/name=calico-node


实战案例:
	1.编写资源清单
[root@k8s-cluster241 ~]# cat > deploy-apps.yaml  <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: lkl-app01
spec:
  replicas: 1
  selector:
    matchLabels:
      apps: v1 
  template:
    metadata:
      labels:
        apps: v1
    spec:
      nodeName: k8s-cluster242
      containers:
      - name: c1
        image: registry.cn-hangzhou.aliyuncs.com/lkl-k8s/apps:v1 

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: lkl-app02
spec:
  replicas: 1
  selector:
    matchLabels:
      apps: v1 
  template:
    metadata:
      labels:
        apps: v1
    spec:
      nodeName: k8s-cluster243
      containers:
      - name: c1
        image: registry.cn-hangzhou.aliyuncs.com/lkl-k8s/apps:v2
EOF


	2.创建资源
[root@k8s-cluster241 ~]# kubectl  apply -f deploy-apps.yaml 
deployment.apps/lkl-app01 created
deployment.apps/lkl-app02 created
[root@k8s-cluster241 ~]# 


	3.测试验证
[root@k8s-cluster241 ~]# kubectl get pods -o wide
NAME                                 READY   STATUS    RESTARTS   AGE    IP              NODE             NOMINATED NODE   READINESS GATES
lkl-app01-859798856d-stxxb   1/1     Running   0          2m6s   10.100.126.84   k8s-cluster242   <none>           <none>
lkl-app02-6c68b6f784-vjw22   1/1     Running   0          2m6s   10.100.22.8     k8s-cluster243   <none>           <none>
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# curl 10.100.126.84
<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8"/>
    <title>yinzhengjie apps v1</title>
    <style>
       div img {
          width: 900px;
          height: 600px;
          margin: 0;
       }
    </style>
  </head>

  <body>
    <h1 style="color: green">凡人修仙传 v1 </h1>
    <div>
      <img src="1.jpg">
    <div>
  </body>

</html>
[root@k8s-cluster241 ~]# 
[root@k8s-cluster241 ~]# curl 10.100.22.8
<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8"/>
    <title>yinzhengjie apps v2</title>
    <style>
       div img {
          width: 900px;
          height: 600px;
          margin: 0;
       }
    </style>
  </head>

  <body>
    <h1 style="color: red">凡人修仙传 v2 </h1>
    <div>
      <img src="2.jpg">
    <div>
  </body>

</html>
[root@k8s-cluster241 ~]#

最后更新：