Kubernetes集群搭建之Master配置篇

[root@master-01 ~]# cd /etc/kubernetes/ssl/[root@master-01 ~]#cat << EOF | tee ca-config.json{"signing": {"default": {"expiry": "87600h"},"profiles": {"kubernetes": {"expiry": "87600h","usages": ["signing","key encipherment","server auth","client auth"]}}}}EOF

[root@master-01 ~]#cat << EOF | tee ca-csr.json{"CN": "kubernetes","key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","L": "Hangzhou","ST": "Hangzhou","O": "k8s","OU": "System"}]}EOF

[root@master-01 ssl]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca2019/03/11 13:31:25 [INFO] generating a new CA key and certificate from CSR2019/03/11 13:31:25 [INFO] generate received request2019/03/11 13:31:25 [INFO] received CSR2019/03/11 13:31:25 [INFO] generating key: rsa-20482019/03/11 13:31:25 [INFO] encoded CSR2019/03/11 13:31:25 [INFO] signed certificate with serial number 389496824246932488061213260650987853035508114869

cat << EOF | tee server-csr.json{"CN": "kubernetes","hosts": ["10.254.0.1","127.0.0.1","192.168.209.130","192.168.209.131","192.168.209.132","kubernetes","kubernetes.default","kubernetes.default.svc","kubernetes.default.svc.cluster","kubernetes.default.svc.cluster.local"],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","L": "Hangzhou","ST": "Hangzhou","O": "k8s","OU": "System"}]}EOF

[root@master-01 ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server2019/03/11 13:37:15 [INFO] generate received request2019/03/11 13:37:15 [INFO] received CSR2019/03/11 13:37:15 [INFO] generating key: rsa-20482019/03/11 13:37:15 [INFO] encoded CSR2019/03/11 13:37:15 [INFO] signed certificate with serial number 68544682773675117056543742114835667888997082622019/03/11 13:37:15 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable forwebsites. For more information see the Baseline Requirements for the Issuance and Managementof Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);specifically, section 10.2.3 ("Information Requirements").

[root@master-01 ssl]# cat << EOF | tee kube-proxy-csr.json{"CN": "system:kube-proxy","hosts": [],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","L": "Hangzhou","ST": "Hangzhou","O": "k8s","OU": "System"}]}EOF

[root@master-01 ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy2019/03/11 13:38:28 [INFO] generate received request2019/03/11 13:38:28 [INFO] received CSR2019/03/11 13:38:28 [INFO] generating key: rsa-20482019/03/11 13:38:29 [INFO] encoded CSR2019/03/11 13:38:29 [INFO] signed certificate with serial number 732603751396221146326217111143810303159789921742019/03/11 13:38:29 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable forwebsites. For more information see the Baseline Requirements for the Issuance and Managementof Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);specifically, section 10.2.3 ("Information Requirements").[root@master-01 ssl]# lsca-config.json ca-csr.json ca.pem kube-proxy-csr.json kube-proxy.pem server-csr.json server.pemca.csr ca-key.pem kube-proxy.csr kube-proxy-key.pem server.csr server-key.pem

[root@master-01 ~]# tar xf kubernetes-server-linux-amd64.tar.gz [root@master-01 ~]# cd kubernetes/server/bin/[root@master-01 bin]# cp kube-scheduler kube-apiserver kube-controller-manager kubectl kube-proxy kubelet /usr/bin/

[root@master-01 ~]# head -c 16 /dev/urandom | od -An -t x | tr -d ' '88b877ad90a3be4ac34711893262a014[root@master-01 ~]# cat /etc/kubernetes/token.csv88b877ad90a3be4ac34711893262a014,kubelet-bootstrap,10001,"system:kubelet-bootstrap"

[root@master-01 ~]# cat /etc/kubernetes/kube-apiserver KUBE_APISERVER_OPTS="--logtostderr=true \--v=4 \--etcd-servers=https://192.168.209.130:2379,https://192.168.209.131:2379,https://192.168.209.132:2379 \--bind-address=192.168.209.130 \--secure-port=6443 \--advertise-address=192.168.209.130 \--allow-privileged=true \--service-cluster-ip-range=10.254.0.0/16 \--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \--authorization-mode=RBAC,Node \--enable-bootstrap-token-auth \--token-auth-file=/etc/kubernetes/token.csv \--service-node-port-range=30000-50000 \--tls-cert-file=/etc/kubernetes/ssl/server.pem \--tls-private-key-file=/etc/kubernetes/ssl/server-key.pem \--client-ca-file=/etc/kubernetes/ssl/ca.pem \--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \--etcd-cafile=/etc/etcd/ssl/ca.pem \--etcd-certfile=/etc/etcd/ssl/server.pem \--etcd-keyfile=/etc/etcd/ssl/server-key.pem"

[root@master-01 ~]# cat /usr/lib/systemd/system/kube-apiserver.service [Unit]Description=Kubernetes API ServerDocumentation=https://github.com/kubernetes/kubernetes[Service]EnvironmentFile=-/etc/kubernetes/kube-apiserverExecStart=/usr/bin/kube-apiserver $KUBE_APISERVER_OPTSRestart=on-failure[Install]WantedBy=multi-user.target

[root@master-01 ~]# systemctl daemon-reload[root@master-01 ~]# systemctl enable kube-apiserverCreated symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.[root@master-01 ~]# systemctl start kube-apiserver[root@master-01 ~]# systemctl status kube-apiserver● kube-apiserver.service - Kubernetes API ServerLoaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)Active: active (running) since 一 2019-03-11 14:04:02 CST; 8s agoDocs: https://github.com/kubernetes/kubernetesMain PID: 86650 (kube-apiserver)Tasks: 10Memory: 343.7MCGroup: /system.slice/kube-apiserver.service└─86650 /usr/bin/kube-apiserver --logtostderr=true --v=4 --etcd-servers=https://192.168.209.130:2379,https://192.168.209.1...[root@master-01 ~]# ps -ef|grep kube-apiserver root 86650 1 13 14:04 ? 00:00:08 /usr/bin/kube-apiserver --logtostderr=true --v=4 --etcd-servers=https://192.168.209.130:2379,https://192.168.209.131:2379,https://192.168.209.132:2379 --bind-address=192.168.209.130 --secure-port=6443 --advertise-address=192.168.209.130 --allow-privileged=true --service-cluster-ip-range=10.254.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction --authorization-mode=RBAC,Node --enable-bootstrap-token-auth --token-auth-file=/etc/kubernetes/token.csv --service-node-port-range=30000-50000 --tls-cert-file=/etc/kubernetes/ssl/server.pem --tls-private-key-file=/etc/kubernetes/ssl/server-key.pem --client-ca-file=/etc/kubernetes/ssl/ca.pem --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem --etcd-cafile=/etc/etcd/ssl/ca.pem --etcd-certfile=/etc/etcd/ssl/server.pem --etcd-keyfile=/etc/etcd/ssl/server-key.pemroot 86742 26572 0 14:05 pts/0 00:00:00 grep --color=auto kube-apiserver[root@master-01 ~]# netstat -tulpn |grep kube-apiservetcp 0 0 192.168.209.130:6443 0.0.0.0:* LISTEN 86650/kube-apiserve tcp 0 0 127.0.0.1:8080 0.0.0.0:* LISTEN 86650/kube-apiserve

[root@master-01 ~]# cat /etc/kubernetes/kube-scheduler KUBE_SCHEDULER_OPTS="--logtostderr=true --v=4 --master=127.0.0.1:8080 --leader-elect"

[root@master-01 ~]# cat /usr/lib/systemd/system/kube-scheduler.service [Unit]Description=Kubernetes SchedulerDocumentation=https://github.com/kubernetes/kubernetes[Service]EnvironmentFile=-/etc/kubernetes/kube-schedulerExecStart=/usr/bin/kube-scheduler $KUBE_SCHEDULER_OPTSRestart=on-failure[Install]WantedBy=multi-user.target

[root@master-01 ~]# systemctl daemon-reload[root@master-01 ~]# systemctl enable kube-scheduler.service Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.[root@master-01 ~]# systemctl start kube-scheduler.service[root@master-01 ~]# systemctl status kube-scheduler.service● kube-scheduler.service - Kubernetes SchedulerLoaded: loaded (/usr/lib/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled)Active: active (running) since 一 2019-03-11 14:13:06 CST; 4s agoDocs: https://github.com/kubernetes/kubernetesMain PID: 87372 (kube-scheduler)Tasks: 9Memory: 46.4MCGroup: /system.slice/kube-scheduler.service└─87372 /usr/bin/kube-scheduler --logtostderr=true --v=4 --master=127.0.0.1:8080 --leader-elect

[root@master-01 ~]# cat /etc/kubernetes/kube-controller-managerKUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=true \--v=4 \--master=127.0.0.1:8080 \--leader-elect=true \--address=127.0.0.1 \--service-cluster-ip-range=10.254.0.0/16 \--cluster-name=kubernetes \--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \--root-ca-file=/etc/kubernetes/ssl/ca.pem \--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem"

[Unit]Description=Kubernetes Controller ManagerDocumentation=https://github.com/kubernetes/kubernetes[Service]EnvironmentFile=-/etc/kubernetes/kube-controller-managerExecStart=/usr/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTSRestart=on-failure[Install]WantedBy=multi-user.target

[root@master-01 ~]# systemctl daemon-reload[root@master-01 ~]# systemctl enable kube-controller-managerCreated symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.[root@master-01 ~]# systemctl start kube-controller-manager[root@master-01 ~]# systemctl status kube-controller-manager● kube-controller-manager.service - Kubernetes Controller ManagerLoaded: loaded (/usr/lib/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled)Active: active (running) since 一 2019-03-11 14:35:36 CST; 6s agoDocs: https://github.com/kubernetes/kubernetesMain PID: 89052 (kube-controller)Tasks: 8Memory: 51.1MCGroup: /system.slice/kube-controller-manager.service└─89052 /usr/bin/kube-controller-manager --logtostderr=true --v=4 --master=127.0.0.1:8080 --leader-elect=true --address=12.

[root@master-01 bin]# scp kube-apiserver kube-controller-manager kubectl kubelet kube-proxy kube-scheduler 192.168.209.132:/usr/bin/ [root@master-01 bin]# scp kube-apiserver kube-controller-manager kubectl kubelet kube-proxy kube-scheduler 192.168.209.133:/usr/bin/

[root@master-01 kubernetes]# scp -r kube-apiserver kube-controller-manager kube-scheduler ssl/ token.csv 192.168.209.131:/etc/kubernetes/[root@master-01 kubernetes]# scp -r kube-apiserver kube-controller-manager kube-scheduler ssl/ token.csv 192.168.209.132:/etc/kubernetes/

[root@master-01 system]# scp kube-apiserver.service kube-controller-manager.service kube-scheduler.service 192.168.209.131:/usr/lib/systemd/system/[root@master-01 system]# scp kube-apiserver.service kube-controller-manager.service kube-scheduler.service 192.168.209.132:/usr/lib/systemd/system/

# master02[root@master-02 ~]# systemctl daemon-reload[root@master-02 ~]# systemctl enable kube-apiserverCreated symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.[root@master-02 ~]# systemctl start kube-apiserver[root@master-02 ~]# systemctl daemon-reload[root@master-02 ~]# systemctl enable kube-scheduler.service Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.[root@master-02 ~]# systemctl start kube-scheduler.service[root@master-02 ~]# systemctl daemon-reload[root@master-02 ~]# systemctl enable kube-controller-managerCreated symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.[root@master-02 ~]# systemctl start kube-controller-manager# master03[root@master-03 yum.repos.d]# systemctl daemon-reload[root@master-03 yum.repos.d]# systemctl enable kube-apiserverCreated symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service[root@master-03 ~]# systemctl start kube-apiserver[root@master-03 ~]# systemctl daemon-reload[root@master-03 ~]# systemctl enable kube-scheduler.service Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.[root@master-03 ~]# systemctl start kube-scheduler.service[root@master-03 ~]# systemctl daemon-reload[root@master-03 ~]# systemctl enable kube-controller-managerCreated symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.[root@master-03 ~]# systemctl start kube-controller-manager

# master-01[root@master-01 kubernetes]# kubectl get csNAME STATUS MESSAGE ERRORcomponentstatus/controller-manager Healthy ok componentstatus/scheduler Healthy ok componentstatus/etcd-0 Healthy {"health":"true"} componentstatus/etcd-1 Healthy {"health":"true"} componentstatus/etcd-2 Healthy {"health":"true"}# master-02[root@master-02 ~]# kubectl get csNAME STATUS MESSAGE ERRORcomponentstatus/scheduler Healthy ok componentstatus/controller-manager Healthy ok componentstatus/etcd-1 Healthy {"health":"true"} componentstatus/etcd-0 Healthy {"health":"true"} componentstatus/etcd-2 Healthy {"health":"true"} # master-03[root@master-03 yum.repos.d]# kubectl get csNAME STATUS MESSAGE ERRORcomponentstatus/controller-manager Healthy ok componentstatus/scheduler Healthy ok componentstatus/etcd-1 Healthy {"health":"true"} componentstatus/etcd-2 Healthy {"health":"true"} componentstatus/etcd-0 Healthy {"health":"true"}

[root@master-01 kubernetes]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrapclusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created

[root@master-01 kubernetes]# cat environment.sh #!/bin/bash#创建kubelet bootstrapping kubeconfig BOOTSTRAP_TOKEN=88b877ad90a3be4ac34711893262a014KUBE_APISERVER="https://192.168.209.130:6443"#设置集群参数kubectl config set-cluster kubernetes \--certificate-authority=/etc/kubernetes/ssl/ca.pem \--embed-certs=true \--server=${KUBE_APISERVER} \--kubeconfig=bootstrap.kubeconfig#设置客户端认证参数kubectl config set-credentials kubelet-bootstrap \--token=${BOOTSTRAP_TOKEN} \--kubeconfig=bootstrap.kubeconfig# 设置上下文参数kubectl config set-context default \--cluster=kubernetes \--user=kubelet-bootstrap \--kubeconfig=bootstrap.kubeconfig# 设置默认上下文kubectl config use-context default --kubeconfig=bootstrap.kubeconfig#----------------------# 创建kube-proxy kubeconfig文件kubectl config set-cluster kubernetes \--certificate-authority=/etc/kubernetes/ssl/ca.pem \--embed-certs=true \--server=${KUBE_APISERVER} \--kubeconfig=kube-proxy.kubeconfigkubectl config set-credentials kube-proxy \--client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \--client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \--embed-certs=true \--kubeconfig=kube-proxy.kubeconfigkubectl config set-context default \--cluster=kubernetes \--user=kube-proxy \--kubeconfig=kube-proxy.kubeconfigkubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

[root@master-01 kubernetes]# sh environment.sh Cluster "kubernetes" set.User "kubelet-bootstrap" set.Context "default" created.Switched to context "default".Cluster "kubernetes" set.User "kube-proxy" set.Context "default" created.Switched to context "default".

[root@master-01 kubernetes]# cat kubelet.config kind: KubeletConfigurationapiVersion: kubelet.config.k8s.io/v1beta1address: 192.168.209.130port: 10250readOnlyPort: 10255cgroupDriver: cgroupfsclusterDNS: ["10.254.0.10"]clusterDomain: cluster.local.failSwapOn: false"authentication": {"anonymous": {"enabled": true},"webhook": {"enabled": false}}

[root@master-01 kubernetes]# cat kubeletKUBELET_OPTS="--logtostderr=true \--v=4 \--hostname-override=192.168.209.130 \--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \--bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \--config=/etc/kubernetes/kubelet.config \--cert-dir=/etc/kubernetes/ssl \--pod-infra-container-image=hub.test.tech/library/pod-infrastructure:latest"

[root@master-01 kubernetes]# cat /usr/lib/systemd/system/kubelet.service [Unit]Description=Kubernetes KubeletAfter=docker.serviceRequires=docker.service[Service]EnvironmentFile=/etc/kubernetes/kubeletExecStart=/usr/bin/kubelet $KUBELET_OPTSRestart=on-failureKillMode=process[Install]WantedBy=multi-user.target

[root@master-01 kubernetes]# systemctl daemon-reload[root@master-01 kubernetes]# systemctl enable kubelet Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.[root@master-01 kubernetes]# systemctl start kubelet

[root@master-01 kubernetes]# kubectl get csrNAME AGE REQUESTOR CONDITIONnode-csr-n7I-bungHuexT06s8OcuXQmkeGZFU5N4I18m0yFzDpM 17s kubelet-bootstrap Pending

[root@master-01 kubernetes]# kubectl certificate approve node-csr-n7I-bungHuexT06s8OcuXQmkeGZFU5N4I18m0yFzDpMcertificatesigningrequest.certificates.k8s.io/node-csr-n7I-bungHuexT06s8OcuXQmkeGZFU5N4I18m0yFzDpM approved

[root@master-01 kubernetes]# kubectl get csrNAME AGE REQUESTOR CONDITIONnode-csr-n7I-bungHuexT06s8OcuXQmkeGZFU5N4I18m0yFzDpM 2m7s kubelet-bootstrap Approved,Issued

[root@master-01 kubernetes]# kubectl get node -o wideNAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME192.168.209.130 Ready <none> 7m23s v1.13.0 192.168.209.130 <none> CentOS Linux 7 (Core) 3.10.0-957.el7.x86_64 docker://18.9.3

KUBE_PROXY_OPTS="--logtostderr=true \--v=4 \--hostname-override=192.168.209.130 \--cluster-cidr=10.254.0.0/16 \--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig"

[root@master-01 kubernetes]# cat /usr/lib/systemd/system/kube-proxy.service [Unit]Description=Kubernetes ProxyAfter=network.target[Service]EnvironmentFile=-/etc/kubernetes/kube-proxyExecStart=/usr/bin/kube-proxy $KUBE_PROXY_OPTSRestart=on-failure[Install]WantedBy=multi-user.target

[root@master-01 kubernetes]# systemctl daemon-reload [root@master-01 kubernetes]# systemctl enable kube-proxy Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /usr/lib/systemd/system/kube-proxy.service.[root@master-01 kubernetes]# systemctl start kube-proxy[root@master-01 kubernetes]# systemctl status kube-proxy● kube-proxy.service - Kubernetes ProxyLoaded: loaded (/usr/lib/systemd/system/kube-proxy.service; enabled; vendor preset: disabled)Active: active (running) since 一 2019-03-11 15:26:13 CST; 4s agoMain PID: 93067 (kube-proxy)Tasks: 0Memory: 6.6MCGroup: /system.slice/kube-proxy.service‣ 93067 /usr/bin/kube-proxy --logtostderr=true --v=4 --hostname-override=192.168.209.130 --cluster-cidr=10.254.0.0/16 --ku

# master-02[root@master-01 kubernetes]# scp kubelet kubelet.config kubelet.kubeconfig kube-proxy bootstrap.kubeconfig kube-proxy.kubeconfig 192.168.209.131:/etc/kubernetes/# master-03[root@master-01 kubernetes]# scp kubelet kubelet.config kubelet.kubeconfig kube-proxy bootstrap.kubeconfig kube-proxy.kubeconfig 192.168.209.132:/etc/kubernetes/# node-01[root@master-01 kubernetes]# scp kubelet kubelet.config kubelet.kubeconfig kube-proxy bootstrap.kubeconfig kube-proxy.kubeconfig 192.168.209.133:/etc/kubernetes/

# master-02[root@master-01 kubernetes]# cd /usr/lib/systemd/system/[root@master-01 system]# scp kube 192.168.209.131:/usr/lib/systemd/system/# master-03[root@master-01 system]# scp kube 192.168.209.132:/usr/lib/systemd/system/# node-01[root@master-01 system]# scp kube 192.168.209.132:/usr/lib/systemd/system/

[root@master-01 kubernetes]# kubectl get csrNAME AGE REQUESTOR CONDITIONnode-csr-S0rYvGBacHcoFPmJrYsqHKTz4d_Yz6KJfpD9yKqsMVY 2m20s kubelet-bootstrap Pendingnode-csr-giUaUWQtl39ZfcrfWmQmKtm5tUjMUddEUX-qJX198Mk 4m9s kubelet-bootstrap Pendingnode-csr-rXqLehzmkMQlXuql_NLVm_MlTxnow9c5QoZLpwS283g 27s kubelet-bootstrap Pending

[root@master-01 kubernetes]# kubectl certificate approve node-csr-S0rYvGBacHcoFPmJrYsqHKTz4d_Yz6KJfpD9yKqsMVYcertificatesigningrequest.certificates.k8s.io/node-csr-S0rYvGBacHcoFPmJrYsqHKTz4d_Yz6KJfpD9yKqsMVY approved[root@master-01 kubernetes]# kubectl certificate approve node-csr-giUaUWQtl39ZfcrfWmQmKtm5tUjMUddEUX-qJX198Mkcertificatesigningrequest.certificates.k8s.io/node-csr-giUaUWQtl39ZfcrfWmQmKtm5tUjMUddEUX-qJX198Mk approved[root@master-01 kubernetes]# kubectl certificate approve node-csr-rXqLehzmkMQlXuql_NLVm_MlTxnow9c5QoZLpwS283gcertificatesigningrequest.certificates.k8s.io/node-csr-rXqLehzmkMQlXuql_NLVm_MlTxnow9c5QoZLpwS283g approved

[root@master-01 ~]# kubectl describe csr node-csr-S0rYvGBacHcoFPmJrYsqHKTz4d_Yz6KJfpD9yKqsMVYName: node-csr-S0rYvGBacHcoFPmJrYsqHKTz4d_Yz6KJfpD9yKqsMVYLabels: <none>Annotations: <none>CreationTimestamp: Mon, 11 Mar 2019 17:20:08 +0800Requesting User: kubelet-bootstrapStatus: Approved,IssuedSubject:Common Name: system:node:192.168.209.132Serial Number: Organization: system:nodesEvents: <none>

[root@master-01 kubernetes]# kubectl get nodeNAME STATUS ROLES AGE VERSION192.168.209.130 Ready <none> 136m v1.13.0192.168.209.131 Ready <none> 104s v1.13.0192.168.209.132 Ready <none> 69s v1.13.0192.168.209.133 Ready <none> 15s v1.13.0

[root@master-01 ~]# cat csr-crb.yaml # Approve all CSRs for the group "system:bootstrappers"kind: ClusterRoleBindingapiVersion: rbac.authorization.k8s.io/v1metadata:name: auto-approve-csrs-for-groupsubjects:- kind: Groupname: system:bootstrappersapiGroup: rbac.authorization.k8s.ioroleRef:kind: ClusterRolename: system:certificates.k8s.io:certificatesigningrequests:nodeclientapiGroup: rbac.authorization.k8s.io---# To let a node of the group "system:nodes" renew its own credentialskind: ClusterRoleBindingapiVersion: rbac.authorization.k8s.io/v1metadata:name: node-client-cert-renewalsubjects:- kind: Groupname: system:nodesapiGroup: rbac.authorization.k8s.ioroleRef:kind: ClusterRolename: system:certificates.k8s.io:certificatesigningrequests:selfnodeclientapiGroup: rbac.authorization.k8s.io---# A ClusterRole which instructs the CSR approver to approve a node requesting a# serving cert matching its client cert.kind: ClusterRoleapiVersion: rbac.authorization.k8s.io/v1metadata:name: approve-node-server-renewal-csrrules:- apiGroups: ["certificates.k8s.io"]resources: ["certificatesigningrequests/selfnodeserver"]verbs: ["create"]---# To let a node of the group "system:nodes" renew its own server credentialskind: ClusterRoleBindingapiVersion: rbac.authorization.k8s.io/v1metadata:name: node-server-cert-renewalsubjects:- kind: Groupname: system:nodesapiGroup: rbac.authorization.k8s.ioroleRef:kind: ClusterRolename: approve-node-server-renewal-csrapiGroup: rbac.authorization.k8s.io

[root@master-01 ~]# kubectl apply -f csr-srb.yaml clusterrolebinding.rbac.authorization.k8s.io/auto-approve-csrs-for-group createdclusterrolebinding.rbac.authorization.k8s.io/node-client-cert-renewal createdclusterrole.rbac.authorization.k8s.io/approve-node-server-renewal-csr createdclusterrolebinding.rbac.authorization.k8s.io/node-server-cert-renewal created

# run一个podkubectl run nginx --image=nginx --port=80 --replicas=1# 为这个pod开一个对外服务端口kubectl expose deploy nginx --port=80 --target-port=80 --type=NodePort

[root@master-01 kubernetes]# kubectl get pod,svcNAME READY STATUS RESTARTS AGEpod/nginx-7899755b7-7s8fl 1/1 Running 0 13mNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGEservice/kubernetes ClusterIP 10.254.0.1 <none> 443/TCP 4h45mservice/nginx NodePort 10.254.58.237 <none> 80:34542/TCP 6m23s