$ wget https://dl.google.com/go/go1.13.8.linux-amd64.tar.gz
$ tar -zxvf go1.13.8.linux-amd64.tar.gz -C /usr/local/
export GOROOT=/usr/local/go
# GOPATH
export GOPATH=$HOME/go
# GOROOT bin
export PATH=$PATH:$GOROOT/bin
# GOPATH bin
export PATH=$PATH:$GOPATH/bin
$ source ~/.bashrc
$ git clone https://github.com/kubernetes/kubernetes.git
$ make KUBE_BUILD_PLATFORMS=linux/amd64
+++ [0215 22:16:44] Building go targets for linux/amd64:
./vendor/k8s.io/code-generator/cmd/deepcopy-gen
+++ [0215 22:16:52] Building go targets for linux/amd64:
./vendor/k8s.io/code-generator/cmd/defaulter-gen
+++ [0215 22:17:00] Building go targets for linux/amd64:
./vendor/k8s.io/code-generator/cmd/conversion-gen
+++ [0215 22:17:12] Building go targets for linux/amd64:
./vendor/k8s.io/kube-openapi/cmd/openapi-gen
+++ [0215 22:17:25] Building go targets for linux/amd64:
./vendor/github.com/go-bindata/go-bindata/go-bindata
+++ [0215 22:17:27] Building go targets for linux/amd64:
cmd/kube-proxy
cmd/kube-apiserver
cmd/kube-controller-manager
cmd/kubelet
cmd/kubeadm
cmd/kube-scheduler
vendor/k8s.io/apiextensions-apiserver
cluster/gce/gci/mounter
cmd/kubectl
cmd/gendocs
cmd/genkubedocs
cmd/genman
cmd/genyaml
cmd/genswaggertypedocs
cmd/linkcheck
vendor/github.com/onsi/ginkgo/ginkgo
test/e2e/e2e.test
cluster/images/conformance/go-runner
cmd/kubemark
vendor/github.com/onsi/ginkgo/ginkgo
$ pwd
/root/Coding/kubernetes/_output/local/bin/linux/amd64
$ ls
apiextensions-apiserver genman go-runner kube-scheduler kubemark
e2e.test genswaggertypedocs kube-apiserver kubeadm linkcheck
gendocs genyaml kube-controller-manager kubectl mounter
genkubedocs ginkgo kube-proxy kubelet
$ git clone https://github.com/wangao1236/k8s_cluster_deploy.git
$ cd k8s_cluster_deploy/scripts
$ chmod +x *.sh
$ mkdir -p k8s/scripts
$ cp k8s_cluster_deploy/scripts/* k8s/scripts
$ curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
$ curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
$ curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
$ chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo
$ cat k8s_cluster_deploy/scripts/cfssl.sh
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo
$ mkdir -p /opt/etcd/{cfg,bin,ssl}
$ wget https://github.com/etcd-io/etcd/releases/download/v3.3.18/etcd-v3.3.18-linux-amd64.tar.gz
$ tar -zxvf etcd-v3.3.18-linux-amd64.tar.gz
$ cp etcd-v3.3.18-linux-amd64/etcdctl etcd-v3.3.18-linux-amd64/etcd /opt/etcd/bin
$ mkdir -p k8s/etcd-cert
$ cp k8s/scripts/etcd-cert.sh k8s/etcd-cert
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat > ca-csr.json <<EOF
{
"CN": "etcd CA",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing"
}
]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
#-----------------------
cat > server-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.1.65",
"192.168.1.66",
"192.168.1.67",
"192.168.1.68",
"192.168.1.69"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
$ ./etcd-cert.sh
2020/02/20 17:18:09 [INFO] generating a new CA key and certificate from CSR
2020/02/20 17:18:09 [INFO] generate received request
2020/02/20 17:18:09 [INFO] received CSR
2020/02/20 17:18:09 [INFO] generating key: rsa-2048
2020/02/20 17:18:09 [INFO] encoded CSR
2020/02/20 17:18:09 [INFO] signed certificate with serial number 712703952401219579947544408367305212876133158662
2020/02/20 17:18:09 [INFO] generate received request
2020/02/20 17:18:09 [INFO] received CSR
2020/02/20 17:18:09 [INFO] generating key: rsa-2048
2020/02/20 17:18:09 [INFO] encoded CSR
2020/02/20 17:18:09 [INFO] signed certificate with serial number 59975233056205858127163767550140095337822886214
2020/02/20 17:18:09 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
$ cp *.pem /opt/etcd/ssl
$ ./k8s/scripts/etcd.sh etcd01 192.168.1.67 etcd01=https://192.168.1.67:2380,etcd02=https://192.168.1.68:2380,etcd03=https://192.168.1.69:2380
#!/bin/bash
# example: ./etcd.sh etcd01 192.168.1.10 etcd01=https://192.168.1.10:2380,etcd02=https://192.168.1.11:2380,etcd03=https://192.168.1.12:2380
ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3
systemctl stop etcd
systemctl disable etcd
WORK_DIR=/opt/etcd
cat <<EOF >$WORK_DIR/cfg/etcd
#[Member]
ETCD_NAME="${ETCD_NAME}"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_LISTEN_CLIENT_URLS="https://${ETCD_IP}:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_IP}:2379"
ETCD_INITIAL_CLUSTER="${ETCD_CLUSTER}"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
cat <<EOF >/usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=${WORK_DIR}/cfg/etcd
ExecStart=${WORK_DIR}/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=${WORK_DIR}/ssl/server.pem \
--key-file=${WORK_DIR}/ssl/server-key.pem \
--peer-cert-file=${WORK_DIR}/ssl/server.pem \
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd
$ scp -r /opt/etcd/ root@192.168.1.68:/opt/
$ scp -r /opt/etcd/ root@192.168.1.69:/opt/
$ scp /usr/lib/systemd/system/etcd.service root@192.168.1.68:/usr/lib/systemd/system/
$ scp /usr/lib/systemd/system/etcd.service root@192.168.1.69:/usr/lib/systemd/system/
[root@192.168.1.68] $ vim /opt/etcd/cfg/etcd
#[Member]
ETCD_NAME="etcd02"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.68:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.68:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.68:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.68:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.1.67:2380,etcd02=https://192.168.1.68:2380,etcd03=https://192.168.1.69
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
[root@192.168.1.69] $ vim /opt/etcd/cfg/etcd
#[Member]
ETCD_NAME="etcd03"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.69:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.69:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.69:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.69:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.1.67:2380,etcd02=https://192.168.1.68:2380,etcd03=https://192.168.1.69
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
$ sudo systemctl enable etcd.service
Created symlink /etc/systemd/system/multi-user.target.wants/etcd.service → /usr/lib/systemd/system/etcd.service.
$ sudo systemctl start etcd.service
$ sudo etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379" cluster-health
member 3143a1397990e241 is healthy: got healthy result from https://192.168.1.68:2379
member 469e7b2757c25086 is healthy: got healthy result from https://192.168.1.67:2379
member 5b1e32d0ab5e3e1b is healthy: got healthy result from https://192.168.1.69:2379
cluster is healthy
$ /opt/etcd/bin/etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://127.0.0.1:2379" set /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}
$ /opt/etcd/bin/etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://127.0.0.1:2379" get /coreos.com/network/config
{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}
$ wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
$ tar -zxvf flannel-v0.11.0-linux-amd64.tar.gz
$ mkdir -p /opt/kubernetes/{cfg,bin,ssl}
$ mv mk-docker-opts.sh flanneld /opt/kubernetes/bin/
$ ./k8s/scripts/flannel.sh https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379
$ cat ./k8s/scripts/flannel.sh
#!/bin/bash
ETCD_ENDPOINTS=${1:-"http://127.0.0.1:2379"}
systemctl stop flanneld
systemctl disable flanneld
cat <<EOF >/opt/kubernetes/cfg/flanneld
FLANNEL_OPTIONS="--etcd-endpoints=${ETCD_ENDPOINTS} \\
-etcd-cafile=/opt/etcd/ssl/ca.pem \\
-etcd-certfile=/opt/etcd/ssl/server.pem \\
-etcd-keyfile=/opt/etcd/ssl/server-key.pem"
EOF
cat <<EOF >/usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker -f /run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable flanneld
systemctl restart flanneld
$ cat /run/flannel/subnet.envFLANNEL_NETWORK=172.17.0.0/16
FLANNEL_SUBNET=172.17.89.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true
$ cat /run/flannel/docker
DOCKER_OPT_BIP="--bip=172.17.89.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=false"
DOCKER_OPT_MTU="--mtu=1450"
DOCKER_OPTS=" --bip=172.17.89.1/24 --ip-masq=false --mtu=1450"
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
BindsTo=containerd.service
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=docker.socket
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
EnvironmentFile=/run/flannel/docker
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H unix:///var/run/docker.soc
#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.soc
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
......
$ systemctl daemon-reload
$ systemctl restart docker
$ ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 172.17.89.1 netmask 255.255.255.0 broadcast 172.17.89.255
ether 02:42:fb:16:3b:12 txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
enp0s3: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.0.2.15 netmask 255.255.255.0 broadcast 10.0.2.255
inet6 fe80::a00:27ff:feaf:b59f prefixlen 64 scopeid 0x20<link>
ether 08:00:27:af:b5:9f txqueuelen 1000 (Ethernet)
RX packets 517 bytes 247169 (247.1 KB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 361 bytes 44217 (44.2 KB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
enp0s8: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.1.67 netmask 255.255.255.0 broadcast 192.168.1.255
inet6 fe80::a00:27ff:fe9f:cb5c prefixlen 64 scopeid 0x20<link>
inet6 2409:8a10:2e24:d130:a00:27ff:fe9f:cb5c prefixlen 64 scopeid 0x0<global>
ether 08:00:27:9f:cb:5c txqueuelen 1000 (Ethernet)
RX packets 9244 bytes 2349434 (2.3 MB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 7420 bytes 1047863 (1.0 MB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 172.17.89.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::60c3:ecff:fe34:9d6c prefixlen 64 scopeid 0x20<link>
ether 62:c3:ec:34:9d:6c txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 6 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1000 (Local Loopback)
RX packets 3722 bytes 904859 (904.8 KB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 3722 bytes 904859 (904.8 KB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@adf9fc37d171 /]# yum install -y net-tools
[root@adf9fc37d171 /]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 172.17.89.2 netmask 255.255.255.0 broadcast 172.17.89.255
ether 02:42:ac:11:59:02 txqueuelen 0 (Ethernet)
RX packets 1538 bytes 14149689 (13.4 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1383 bytes 81403 (79.4 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
loop txqueuelen 1000 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@adf9fc37d171 /]# ping 172.17.89.1
PING 172.17.89.1 (172.17.89.1) 56(84) bytes of data.
64 bytes from 172.17.89.1: icmp_seq=1 ttl=64 time=0.045 ms
64 bytes from 172.17.89.1: icmp_seq=2 ttl=64 time=0.045 ms
64 bytes from 172.17.89.1: icmp_seq=3 ttl=64 time=0.050 ms
64 bytes from 172.17.89.1: icmp_seq=4 ttl=64 time=0.052 ms
64 bytes from 172.17.89.1: icmp_seq=5 ttl=64 time=0.049 ms
$ sudo apt-get -y install haproxy keepalived
listen admin_stats
bind 0.0.0.0:10080
mode http
log 127.0.0.1 local0 err
stats refresh 30s
stats uri /status
stats realm welcome login\ Haproxy
stats auth admin:123456
stats hide-version
stats admin if TRUE
listen kube-master
bind 0.0.0.0:8443
mode tcp
option tcplog
balance source
server 192.168.1.67 192.168.1.67:6443 check inter 2000 fall 2 rise 2 weight 1
server 192.168.1.68 192.168.1.68:6443 check inter 2000 fall 2 rise 2 weight 1
server 192.168.1.69 192.168.1.69:6443 check inter 2000 fall 2 rise 2 weight 1
$ cat /etc/haproxy/haproxy.cfg
global
log /dev/log local0
log /dev/log local1 notice
chroot /var/lib/haproxy
stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
stats timeout 30s
user haproxy
group haproxy
daemon
# Default SSL material locations
ca-base /etc/ssl/certs
crt-base /etc/ssl/private
# Default ciphers to use on SSL-enabled listening sockets.
# For more information, see ciphers(1SSL). This list is from:
# https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
# An alternative list with additional directives can be obtained from
# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=haproxy
ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
ssl-default-bind-options no-sslv3
defaults
log global
mode http
option httplog
option dontlognull
timeout connect 5000
timeout client 50000
timeout server 50000
errorfile 400 /etc/haproxy/errors/400.http
errorfile 403 /etc/haproxy/errors/403.http
errorfile 408 /etc/haproxy/errors/408.http
errorfile 500 /etc/haproxy/errors/500.http
errorfile 502 /etc/haproxy/errors/502.http
errorfile 503 /etc/haproxy/errors/503.http
errorfile 504 /etc/haproxy/errors/504.http
listen admin_stats
bind 0.0.0.0:10080
mode http
log 127.0.0.1 local0 err
stats refresh 30s
stats uri /status
stats realm welcome login\ Haproxy
stats auth admin:123456
stats hide-version
stats admin if TRUE
listen kube-master
bind 0.0.0.0:8443
mode tcp
option tcplog
balance source
server 192.168.1.67 192.168.1.67:6443 check inter 2000 fall 2 rise 2 weight 1
server 192.168.1.68 192.168.1.68:6443 check inter 2000 fall 2 rise 2 weight 1
server 192.168.1.69 192.168.1.69:6443 check inter 2000 fall 2 rise 2 weight 1
$ sudo systemctl enable haproxy
$ sudo systemctl daemon-reload
$ sudo systemctl restart haproxy.service
$ sudo vim /etc/keepalived/keepalived.conf
global_defs {
router_id lb-master
}
vrrp_script check-haproxy {
script "killall -0 haproxy"
interval 5
weight -30
}
vrrp_instance VI-kube-master {
state MASTER
priority 120
dont_track_primary
interface enp0s3
virtual_router_id 68
advert_int 3
track_script {
check-haproxy
}
virtual_ipaddress {
192.168.1.99
}
}
$ sudo vim /etc/keepalived/keepalived.conf
global_defs {
router_id lb-backup
}
vrrp_script check-haproxy {
script "killall -0 haproxy"
interval 5
weight -30
}
vrrp_instance VI-kube-master {
state BACKUP
priority 110
dont_track_primary
interface enp0s3
virtual_router_id 68
advert_int 3
track_script {
check-haproxy
}
virtual_ipaddress {
192.168.1.99
}
}
$ sudo systemctl enable keepalived
$ sudo systemctl daemon-reload
$ sudo systemctl restart keepalived.service
$ sudo mkdir -p /opt/kubernetes/{ssl,cfg,bin,log}
cat > kube-apiserver-csr.json <<EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.1.99", // 虚拟机 IP
"192.168.1.67", // master1 IP
"192.168.1.68", // master2 IP
"192.168.1.69", // master3 IP
"10.254.0.1", // kube-apiserver 定义的 --service-cluster-ip-range 参数指定的 IP 地址段(10.254.0.0/24)的第一个IP地址
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
cat > kube-controller-manager-csr.json <<EOF
{
"CN": "system:kube-controller-manager",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"192.168.1.67",
"192.168.1.68",
"192.168.1.69"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:kube-controller-manager",
"OU": "System"
}
]
}
EOF
cat > kube-scheduler-csr.json <<EOF
{
"CN": "system:kube-scheduler",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"192.168.1.67",
"192.168.1.68",
"192.168.1.69"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:kube-scheduler",
"OU": "System"
}
]
}
EOF
rm -rf master
rm -rf node
mkdir master
mkdir node
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat > ca-csr.json <<EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
#----------------------- kube-apiserver
echo "generate kube-apiserver cert"
cd master
cat > kube-apiserver-csr.json <<EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.1.99",
"192.168.1.67",
"192.168.1.68",
"192.168.1.69",
"10.254.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
cfssl gencert -ca=../ca.pem -ca-key=../ca-key.pem -config=../ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver
cd ..
#----------------------- kubectl
echo "generate kubectl cert"
cat > admin-csr.json <<EOF
{
"CN": "admin",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
#----------------------- kube-controller-manager
echo "generate kube-controller-manager cert"
cd master
cat > kube-controller-manager-csr.json <<EOF
{
"CN": "system:kube-controller-manager",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"192.168.1.67",
"192.168.1.68",
"192.168.1.69"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:kube-controller-manager",
"OU": "System"
}
]
}
EOF
cfssl gencert -ca=../ca.pem -ca-key=../ca-key.pem -config=../ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
cd ..
#----------------------- kube-scheduler
echo "generate kube-scheduler cert"
cd master
cat > kube-scheduler-csr.json <<EOF
{
"CN": "system:kube-scheduler",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"192.168.1.67",
"192.168.1.68",
"192.168.1.69"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:kube-scheduler",
"OU": "System"
}
]
}
EOF
cfssl gencert -ca=../ca.pem -ca-key=../ca-key.pem -config=../ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
cd ..
#----------------------- kube-proxy
echo "generate kube-proxy cert"
cd node
cat > kube-proxy-csr.json <<EOF
{
"CN": "system:kube-proxy",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:kube-proxy",
"OU": "System"
}
]
}
EOF
cfssl gencert -ca=../ca.pem -ca-key=../ca-key.pem -config=../ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
cd ..
$ mkdir -p k8s/k8s-cert
$ cp k8s/scripts/k8s-cert.sh k8s/k8s-cert
$ cd k8s/k8s-cert
$ ./k8s-cert.sh
$ sudo cp -r ca* kube* master node /opt/kubernetes/ssl/
$ sudo scp -r /opt/kubernetes/ssl/* root@192.168.1.68:/opt/kubernetes/ssl/
$ sudo scp -r /opt/kubernetes/ssl/* root@192.168.1.69:/opt/kubernetes/ssl/
$ mkdir -p ~/.kube
$ mkdir -p k8s/kubeconfig
$ cp k8s/scripts/kubeconfig.sh k8s/kubeconfig
$ cd k8s/kubeconfig
$ sudo ./kubeconfig.sh 192.168.1.99 /opt/kubernetes/ssl
0524b3077444a437dfc662e5739bfa1a
===> generate kubectl config
Cluster "kubernetes" set.
User "admin" set.
Context "admin@kubernetes" created.
Switched to context "admin@kubernetes".
===> generate kube-controller-manager.kubeconfig
Cluster "kubernetes" set.
User "system:kube-controller-manager" set.
Context "system:kube-controller-manager@kubernetes" created.
Switched to context "system:kube-controller-manager@kubernetes".
===> generate kube-scheduler.kubeconfig
Cluster "kubernetes" set.
User "system:kube-scheduler" set.
Context "system:kube-scheduler@kubernetes" created.
Switched to context "system:kube-scheduler@kubernetes".
===> generate kubelet bootstrapping kubeconfig
Cluster "kubernetes" set.
User "kubelet-bootstrap" set.
Context "default" created.
Switched to context "default".
===> generate kube-proxy.kubeconfig
Cluster "kubernetes" set.
User "system:kube-proxy" set.
Context "system:kube-proxy@kubernetes" created.
Switched to context "system:kube-proxy@kubernetes".
#----------------------创建 kube-apiserver TLS Bootstrapping Token
BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
echo ${BOOTSTRAP_TOKEN}
cat > token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
#----------------------
APISERVER=$1
SSL_DIR=$2
export KUBE_APISERVER="https://$APISERVER:8443"
#----------------------
echo "===> generate kubectl config"
# 创建 kubectl config
kubectl config set-cluster kubernetes \
--certificate-authority=$SSL_DIR/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=config
kubectl config set-credentials admin \
--client-certificate=$SSL_DIR/admin.pem \
--client-key=$SSL_DIR/admin-key.pem \
--embed-certs=true \
--kubeconfig=config
kubectl config set-context admin@kubernetes \
--cluster=kubernetes \
--user=admin \
--kubeconfig=config
kubectl config use-context admin@kubernetes --kubeconfig=config
#----------------------
echo "===> generate kube-controller-manager.kubeconfig"
# 创建 kube-controller-manager.kubeconfig
kubectl config set-cluster kubernetes \
--certificate-authority=$SSL_DIR/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-credentials system:kube-controller-manager \
--client-certificate=$SSL_DIR/master/kube-controller-manager.pem \
--client-key=$SSL_DIR/master/kube-controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-context system:kube-controller-manager@kubernetes \
--cluster=kubernetes \
--user=system:kube-controller-manager \
--kubeconfig=kube-controller-manager.kubeconfig
kubectl config use-context system:kube-controller-manager@kubernetes --kubeconfig=kube-controller-manager.kubeconfig
#----------------------
echo "===> generate kube-scheduler.kubeconfig"
# 创建 kube-scheduler.kubeconfig
kubectl config set-cluster kubernetes \
--certificate-authority=$SSL_DIR/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-scheduler.kubeconfig
kubectl config set-credentials system:kube-scheduler \
--client-certificate=$SSL_DIR/master/kube-scheduler.pem \
--client-key=$SSL_DIR/master/kube-scheduler-key.pem \
--embed-certs=true \
--kubeconfig=kube-scheduler.kubeconfig
kubectl config set-context system:kube-scheduler@kubernetes \
--cluster=kubernetes \
--user=system:kube-scheduler \
--kubeconfig=kube-scheduler.kubeconfig
kubectl config use-context system:kube-scheduler@kubernetes --kubeconfig=kube-scheduler.kubeconfig
#----------------------
echo "===> generate kubelet bootstrapping kubeconfig"
# 创建 kubelet bootstrapping kubeconfig
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=$SSL_DIR/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=bootstrap.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=bootstrap.kubeconfig
# 设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
#----------------------
echo "===> generate kube-proxy.kubeconfig"
# 创建 kube-proxy.kubeconfig
kubectl config set-cluster kubernetes \
--certificate-authority=$SSL_DIR/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.kubeconfig
kubectl config set-credentials system:kube-proxy \
--client-certificate=$SSL_DIR/node/kube-proxy.pem \
--client-key=$SSL_DIR/node/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig
kubectl config set-context system:kube-proxy@kubernetes \
--cluster=kubernetes \
--user=system:kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
kubectl config use-context system:kube-proxy@kubernetes --kubeconfig=kube-proxy.kubeconfig
$ sudo chown ao:ao config
$ cp config ~/.kube
$ scp config ao@192.168.1.68:/home/ao/.kube
$ scp config ao@192.168.1.69:/home/ao/.kube
$ sudo cp token.csv /opt/kubernetes/cfg
$ sudo scp token.csv root@192.168.1.68:/opt/kubernetes/cfg
$ sudo scp token.csv root@192.168.1.69:/opt/kubernetes/cfg
opt/kubernetes/bin/
中
$ cp kube-apiserver kubectl kube-controller-manager kube-scheduler kubelet kube-proxy /opt/kubernetes/bin/
$ sudo ./k8s/scripts/apiserver.sh 192.168.1.67 https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379
$ sudo ./k8s/scripts/apiserver.sh 192.168.1.68 https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379
$ sudo ./k8s/scripts/apiserver.sh 192.168.1.69 https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379
#!/bin/bash
MASTER_ADDRESS=$1
ETCD_SERVERS=$2
systemctl stop kube-apiserver
systemctl disable kube-apiserver
cat <<EOF >/opt/kubernetes/cfg/kube-apiserver
KUBE_APISERVER_OPTS="--logtostderr=true \\
--v=4 \\
--anonymous-auth=false \\
--etcd-servers=${ETCD_SERVERS} \\
--etcd-cafile=/opt/etcd/ssl/ca.pem \\
--etcd-certfile=/opt/etcd/ssl/server.pem \\
--etcd-keyfile=/opt/etcd/ssl/server-key.pem \\
--service-cluster-ip-range=10.254.0.0/16 \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \\
--bind-address=${MASTER_ADDRESS} \\
--secure-port=6443 \\
--client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--token-auth-file=/opt/kubernetes/cfg/token.csv \\
--allow-privileged=true \\
--tls-cert-file=/opt/kubernetes/ssl/master/kube-apiserver.pem \\
--tls-private-key-file=/opt/kubernetes/ssl/master/kube-apiserver-key.pem \\
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--advertise-address=${MASTER_ADDRESS} \\
--authorization-mode=RBAC,Node \\
--kubelet-https=true \\
--enable-bootstrap-token-auth \\
--kubelet-certificate-authority=/opt/kubernetes/ssl/ca.pem \\
--kubelet-client-key=/opt/kubernetes/ssl/master/kube-apiserver-key.pem \\
--kubelet-client-certificate=/opt/kubernetes/ssl/master/kube-apiserver.pem \\
--service-node-port-range=30000-50000"
EOF
cat <<EOF >/usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-apiserver
ExecStart=/opt/kubernetes/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-apiserver
systemctl restart kube-apiserver
$ systemctl status kube-apiserver.service
● kube-apiserver.service - Kubernetes API Server
Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: enabled)
Active: active (running) since Tue 2020-02-25 04:05:07 UTC; 2s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 7025 (kube-apiserver)
Tasks: 11 (limit: 2317)
CGroup: /system.slice/kube-apiserver.service
└─7025 /opt/kubernetes/bin/kube-apiserver --logtostderr=true --v=4 --anonymous-auth=false --etcd-servers
Feb 25 04:05:08 master1 kube-apiserver[7025]: I0225 04:05:08.782348 7025 endpoint.go:68] ccResolverWrapper: send
Feb 25 04:05:08 master1 kube-apiserver[7025]: I0225 04:05:08.782501 7025 reflector.go:211] Listing and watching
Feb 25 04:05:08 master1 kube-apiserver[7025]: I0225 04:05:08.788181 7025 store.go:1362] Monitoring apiservices.a
Feb 25 04:05:08 master1 kube-apiserver[7025]: I0225 04:05:08.789982 7025 watch_cache.go:449] Replace watchCache
Feb 25 04:05:08 master1 kube-apiserver[7025]: I0225 04:05:08.790831 7025 deprecated_insecure_serving.go:53] Serv
Feb 25 04:05:08 master1 kube-apiserver[7025]: I0225 04:05:08.794071 7025 reflector.go:211] Listing and watching
Feb 25 04:05:08 master1 kube-apiserver[7025]: I0225 04:05:08.797572 7025 watch_cache.go:449] Replace watchCache
Feb 25 04:05:09 master1 kube-apiserver[7025]: I0225 04:05:09.026015 7025 client.go:361] parsed scheme: "endpoint
Feb 25 04:05:09 master1 kube-apiserver[7025]: I0225 04:05:09.026087 7025 endpoint.go:68] ccResolverWrapper: send
Feb 25 04:05:09 master1 kube-apiserver[7025]: I0225 04:05:09.888015 7025 aggregator.go:109] Building initial Ope
lines 1-19/19 (END)
$ cat /opt/kubernetes/cfg/kube-apiserver
KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--anonymous-auth=false \
--etcd-servers=https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379 \
--etcd-cafile=/opt/etcd/ssl/ca.pem \
--etcd-certfile=/opt/etcd/ssl/server.pem \
--etcd-keyfile=/opt/etcd/ssl/server-key.pem \
--service-cluster-ip-range=10.254.0.0/16 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--bind-address=192.168.1.67 \
--secure-port=6443 \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--allow-privileged=true \
--tls-cert-file=/opt/kubernetes/ssl/master/kube-apiserver.pem \
--tls-private-key-file=/opt/kubernetes/ssl/master/kube-apiserver-key.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--advertise-address=192.168.1.67 \
--authorization-mode=RBAC,Node \
--kubelet-https=true \
--enable-bootstrap-token-auth \
--kubelet-certificate-authority=/opt/kubernetes/ssl/ca.pem \
--kubelet-client-key=/opt/kubernetes/ssl/master/kube-apiserver-key.pem \
--kubelet-client-certificate=/opt/kubernetes/ssl/master/kube-apiserver.pem \
--service-node-port-range=30000-50000"
$ kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes
$ cd k8s/kubeconfig
$ sudo cp kube-scheduler.kubeconfig /opt/kubernetes/cfg
$ sudo scp kube-scheduler.kubeconfig root@192.168.1.68:/opt/kubernetes/cfg
$ sudo scp kube-scheduler.kubeconfig root@192.168.1.69:/opt/kubernetes/cfg
$ sudo ./k8s/scripts/scheduler.sh
#!/bin/bash
systemctl stop kube-scheduler
systemctl disable kube-scheduler
cat <<EOF >/opt/kubernetes/cfg/kube-scheduler
KUBE_SCHEDULER_OPTS="--logtostderr=true \\
--v=4 \\
--bind-address=0.0.0.0 \\
--port=10251 \\
--secure-port=10259 \\
--kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig \\
--requestheader-client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--authentication-kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig \\
--authorization-kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig \\
--client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--tls-cert-file=/opt/kubernetes/ssl/master/kube-scheduler.pem \\
--tls-private-key-file=/opt/kubernetes/ssl/master/kube-scheduler-key.pem \\
--leader-elect=true"
EOF
cat <<EOF >/usr/lib/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-scheduler
ExecStart=/opt/kubernetes/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-scheduler
systemctl restart kube-scheduler
$ sudo systemctl status kube-scheduler.service
● kube-scheduler.service - Kubernetes Scheduler
Loaded: loaded (/usr/lib/systemd/system/kube-scheduler.service; enabled; vendor preset: enabled)
Active: active (running) since Tue 2020-02-25 04:11:03 UTC; 1min 7s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 7701 (kube-scheduler)
Tasks: 10 (limit: 2317)
CGroup: /system.slice/kube-scheduler.service
└─7701 /opt/kubernetes/bin/kube-scheduler --logtostderr=true --v=4 --bind-address=0.0.0.0 --port=10251 -
Feb 25 04:11:43 master1 kube-scheduler[7701]: I0225 04:11:43.911182 7701 reflector.go:211] Listing and watching
Feb 25 04:11:43 master1 kube-scheduler[7701]: E0225 04:11:43.913844 7701 reflector.go:178] k8s.io/kubernetes/cmd
Feb 25 04:11:46 master1 kube-scheduler[7701]: I0225 04:11:46.438728 7701 reflector.go:211] Listing and watching
Feb 25 04:11:46 master1 kube-scheduler[7701]: E0225 04:11:46.441883 7701 reflector.go:178] k8s.io/client-go/info
Feb 25 04:11:47 master1 kube-scheduler[7701]: I0225 04:11:47.086981 7701 reflector.go:211] Listing and watching
Feb 25 04:11:47 master1 kube-scheduler[7701]: E0225 04:11:47.088902 7701 reflector.go:178] k8s.io/client-go/info
Feb 25 04:12:09 master1 kube-scheduler[7701]: I0225 04:12:09.120429 7701 reflector.go:211] Listing and watching
Feb 25 04:12:09 master1 kube-scheduler[7701]: E0225 04:12:09.123594 7701 reflector.go:178] k8s.io/client-go/info
Feb 25 04:12:09 master1 kube-scheduler[7701]: I0225 04:12:09.788768 7701 reflector.go:211] Listing and watching
Feb 25 04:12:09 master1 kube-scheduler[7701]: E0225 04:12:09.790724 7701 reflector.go:178] k8s.io/client-go/info
lines 1-19/19 (END)
$ cd k8s/kubeconfig
$ sudo cp kube-controller-manager.kubeconfig /opt/kubernetes/cfg
$ sudo scp kube-controller-manager.kubeconfig root@192.168.1.68:/opt/kubernetes/cfg
$ sudo scp kube-controller-manager.kubeconfig root@192.168.1.69:/opt/kubernetes/cfg
$ sudo ./k8s/scripts/controller-manager.sh
#!/bin/bash
systemctl stop kube-controller-manager.service
systemctl disable kube-controller-manager.service
cat <<EOF >/opt/kubernetes/cfg/kube-controller-manager
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=true \\
--v=4 \\
--bind-address=0.0.0.0 \\
--cluster-name=kubernetes \\
--kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig \\
--requestheader-client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--authentication-kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig \\
--authorization-kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig \\
--leader-elect=true \\
--service-cluster-ip-range=10.254.0.0/16 \\
--controllers=*,bootstrapsigner,tokencleaner \\
--tls-cert-file=/opt/kubernetes/ssl/master/kube-controller-manager.pem \\
--tls-private-key-file=/opt/kubernetes/ssl/master/kube-controller-manager-key.pem \\
--cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \\
--cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--root-ca-file=/opt/kubernetes/ssl/ca.pem \\
--service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--secure-port=10257 \\
--use-service-account-credentials=true \\
--experimental-cluster-signing-duration=87600h0m0s"
EOF
#--allocate-node-cidrs=true \\
#--cluster-cidr=172.17.0.0/16 \\
cat <<EOF >/usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-controller-manager
ExecStart=/opt/kubernetes/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl restart kube-controller-manager
$ systemctl status kube-controller-manager.service
● kube-controller-manager.service - Kubernetes Controller Manager
Loaded: loaded (/usr/lib/systemd/system/kube-controller-manager.service; enabled; vendor preset: enabled)
Active: active (running) since Sun 2020-02-23 13:06:29 UTC; 15min ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 22526 (kube-controller)
Tasks: 9 (limit: 2317)
CGroup: /system.slice/kube-controller-manager.service
└─22526 /opt/kubernetes/bin/kube-controller-manager --logtostderr=false --v=3 --bind-address=192.168.1.6
Feb 23 13:06:29 clean systemd[1]: Started Kubernetes Controller Manager.
$ vim ~/.zshrc
......
export PATH=$PATH:/opt/kubernetes/bin/
$ source ~/.zshrc
$ kubectl cluster-info
Kubernetes master is running at https://192.168.1.99:8443
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.
$ kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-1 Healthy {"health":"true"}
etcd-2 Healthy {"health":"true"}
etcd-0 Healthy {"health":"true"}
$ kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
clusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created
$ cd k8s/kubeconfig
$ sudo cp kube-controller-manager.kubeconfig /opt/kubernetes/cfg
$ sudo scp kube-controller-manager.kubeconfig root@192.168.1.68:/opt/kubernetes/cfg
$ sudo scp kube-controller-manager.kubeconfig root@192.168.1.69:/opt/kubernetes/cfg
sudo ./k8s/scripts/kubelet.sh 192.168.1.67 node1
sudo ./k8s/scripts/kubelet.sh 192.168.1.68 node2
sudo ./k8s/scripts/kubelet.sh 192.168.1.69 node3
#!/bin/bash
NODE_ADDRESS=$1
NODE_NAME=$2
DNS_SERVER_IP=${3:-"10.254.0.2"}
systemctl stop kubelet
systemctl disable kubelet
cat <<EOF >/opt/kubernetes/cfg/kubelet
KUBELET_OPTS="--logtostderr=true \\
--v=4 \\
--config=/opt/kubernetes/cfg/kubelet.config \\
--node-ip=${NODE_ADDRESS} \\
--bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \\
--cert-dir=/opt/kubernetes/ssl/node \\
--hostname-override=${NODE_NAME} \\
--node-labels=node.kubernetes.io/k8s-master=true \\
--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \\
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0"
EOF
#--cni-bin-dir=/opt/cni/bin \\
#--cni-conf-dir=/opt/cni/net.d \\
#--network-plugin=cni \\
cat <<EOF >/opt/kubernetes/cfg/kubelet.config
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: ${NODE_ADDRESS}
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- ${DNS_SERVER_IP}
clusterDomain: cluster.local.
failSwapOn: false
authentication:
anonymous:
enabled: false
webhook:
enabled: true
x509:
clientCAFile: "/opt/kubernetes/ssl/ca.pem"
authorization:
mode: Webhook
EOF
cat <<EOF >/usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kubelet
ExecStart=/opt/kubernetes/bin/kubelet \$KUBELET_OPTS
Restart=on-failure
KillMode=process
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet
$ kubectl get csr
NAME AGE REQUESTOR CONDITION
node-csr-HHJMkN9RvwkgTkWGJThtsIPPlexh1Ci5vyOcjEhwk5c 29s kubelet-bootstrap Pending
node-csr-Ih-JtbfHPzP8u0_YI0By7RWMPCEfaEpapi47kil1YbU 4s kubelet-bootstrap Pending
node-csr-eyb0y_uxEWgPHnUQ2DyEhCK09AkirUp11O3b40zFyAQ 1s kubelet-bootstrap Pending
$ kubectl certificate approve node-csr-HHJMkN9RvwkgTkWGJThtsIPPlexh1Ci5vyOcjEhwk5c node-csr-Ih-JtbfHPzP8u0_YI0By7RWMPCEfaEpapi47kil1YbU node-csr-eyb0y_uxEWgPHnUQ2DyEhCK09AkirUp11O3b40zFyAQ
certificatesigningrequest.certificates.k8s.io/node-csr-HHJMkN9RvwkgTkWGJThtsIPPlexh1Ci5vyOcjEhwk5c approved
certificatesigningrequest.certificates.k8s.io/node-csr-Ih-JtbfHPzP8u0_YI0By7RWMPCEfaEpapi47kil1YbU approved
certificatesigningrequest.certificates.k8s.io/node-csr-eyb0y_uxEWgPHnUQ2DyEhCK09AkirUp11O3b40zFyAQ approved
$ kubectl get csr
NAME AGE REQUESTOR CONDITION
node-csr-HHJMkN9RvwkgTkWGJThtsIPPlexh1Ci5vyOcjEhwk5c 3m37s kubelet-bootstrap Approved,Issued
node-csr-Ih-JtbfHPzP8u0_YI0By7RWMPCEfaEpapi47kil1YbU 3m12s kubelet-bootstrap Approved,Issued
node-csr-eyb0y_uxEWgPHnUQ2DyEhCK09AkirUp11O3b40zFyAQ 3m9s kubelet-bootstrap Approved,Issued
$ kubectl get node
NAME STATUS ROLES AGE VERSION
node1 Ready <none> 2m36s v1.18.0-alpha.5.158+1c60045db0bd6e
node2 Ready <none> 2m36s v1.18.0-alpha.5.158+1c60045db0bd6e
node3 Ready <none> 2m36s v1.18.0-alpha.5.158+1c60045db0bd6e
$ kubectl label node node1 node2 node3 node-role.kubernetes.io/master=true
node/node1 labeled
node/node2 labeled
node/node3 labeled
$ kubectl get node
NAME STATUS ROLES AGE VERSION
node1 Ready master 3m36s v1.18.0-alpha.5.158+1c60045db0bd6e
node2 Ready master 3m36s v1.18.0-alpha.5.158+1c60045db0bd6e
node3 Ready master 3m36s v1.18.0-alpha.5.158+1c60045db0bd6e
$ kubectl taint nodes --all node-role.kubernetes.io/master=true:NoSchedule
$ kubectl taint nodes node1 node2 node3 node-role.kubernetes.io/master-
$ cd k8s/kubeconfig
$ sudo cp kube-proxy.kubeconfig /opt/kubernetes/cfg
$ sudo scp kube-proxy.kubeconfig root@192.168.1.68:/opt/kubernetes/cfg
$ sudo scp kube-proxy.kubeconfig root@192.168.1.69:/opt/kubernetes/cfg
$ ./k8s/scripts/proxy.sh node1
$ ./k8s/scripts/proxy.sh node2
$ ./k8s/scripts/proxy.sh node3
#!/bin/bash
NODE_NAME=$1
systemctl stop kube-proxy
systemctl disable kube-proxy
cat <<EOF >/opt/kubernetes/cfg/kube-proxy
KUBE_PROXY_OPTS="--logtostderr=true \\
--v=4 \\
--bind-address=0.0.0.0 \\
--hostname-override=${NODE_NAME} \\
--cleanup-ipvs=true \\
--cluster-cidr=10.254.0.0/16 \\
--proxy-mode=ipvs \\
--ipvs-min-sync-period=5s \\
--ipvs-sync-period=5s \\
--ipvs-scheduler=wrr \\
--masquerade-all=true \\
--kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig"
EOF
cat <<EOF >/usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Proxy
After=network.target
[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-proxy
ExecStart=/opt/kubernetes/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable kube-proxy
systemctl restart kube-proxy
$ sudo systemctl status kube-proxy.service
● kube-proxy.service - Kubernetes Proxy
Loaded: loaded (/usr/lib/systemd/system/kube-proxy.service; enabled; vendor preset: enabled)
Active: active (running) since Tue 2020-02-25 07:32:58 UTC; 12s ago
Main PID: 30924 (kube-proxy)
Tasks: 7 (limit: 2317)
CGroup: /system.slice/kube-proxy.service
└─30924 /opt/kubernetes/bin/kube-proxy --logtostderr=true --v=4 --bind-address=0.0.0.0 --hostname-overri
Feb 25 07:33:01 master3 kube-proxy[30924]: I0225 07:33:01.877754 30924 config.go:169] Calling handler.OnEndpoints
Feb 25 07:33:02 master3 kube-proxy[30924]: I0225 07:33:02.027867 30924 config.go:169] Calling handler.OnEndpoints
Feb 25 07:33:03 master3 kube-proxy[30924]: I0225 07:33:03.906364 30924 config.go:169] Calling handler.OnEndpoints
Feb 25 07:33:04 master3 kube-proxy[30924]: I0225 07:33:04.058010 30924 config.go:169] Calling handler.OnEndpoints
Feb 25 07:33:05 master3 kube-proxy[30924]: I0225 07:33:05.937519 30924 config.go:169] Calling handler.OnEndpoints
Feb 25 07:33:06 master3 kube-proxy[30924]: I0225 07:33:06.081698 30924 config.go:169] Calling handler.OnEndpoints
Feb 25 07:33:07 master3 kube-proxy[30924]: I0225 07:33:07.970036 30924 config.go:169] Calling handler.OnEndpoints
Feb 25 07:33:08 master3 kube-proxy[30924]: I0225 07:33:08.118982 30924 config.go:169] Calling handler.OnEndpoints
Feb 25 07:33:09 master3 kube-proxy[30924]: I0225 07:33:09.996659 30924 config.go:169] Calling handler.OnEndpoints
Feb 25 07:33:10 master3 kube-proxy[30924]: I0225 07:33:10.148146 30924 config.go:169] Calling handler.OnEndpoints
lines 1-18/18 (END)
$ mkdir -p k8s/yamls
$ cd k8s/yamls
$ vim nginx-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
spec:
selector:
matchLabels:
app: nginx
replicas: 2
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.7.9
ports:
- containerPort: 80
$ kubectl apply -f nginx-deployment.yaml
$ kubectl get pod
NAME READY STATUS RESTARTS AGE
nginx-deployment-54f57cf6bf-6d4n5 1/1 Running 0 5s
nginx-deployment-54f57cf6bf-zzdv4 1/1 Running 0 5s
$ kubectl describe pod nginx-deployment-54f57cf6bf-6d4n5
Name: nginx-deployment-54f57cf6bf-6d4n5
Namespace: default
Priority: 0
Node: node3/192.168.1.69
Start Time: Tue, 25 Feb 2020 07:35:08 +0000
Labels: app=nginx
pod-template-hash=54f57cf6bf
Annotations: <none>
Status: Running
IP: 172.17.89.2
IPs:
IP: 172.17.89.2
Controlled By: ReplicaSet/nginx-deployment-54f57cf6bf
Containers:
nginx:
Container ID: docker://222b1dd1bb57fdd36b4eda31100477531f94a82c844a2f042c444f0a710faf20
Image: nginx:1.7.9
Image ID: docker-pullable://nginx@sha256:e3456c851a152494c3e4ff5fcc26f240206abac0c9d794affb40e0714846c451
Port: 80/TCP
Host Port: 0/TCP
State: Running
Started: Tue, 25 Feb 2020 07:35:10 +0000
Ready: True
Restart Count: 0
Environment: <none>
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from default-token-p92fn (ro)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
default-token-p92fn:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-p92fn
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled <unknown> default-scheduler Successfully assigned default/nginx-deployment-54f57cf6bf-6d4n5 to node3
Normal Pulled 117s kubelet, node3 Container image "nginx:1.7.9" already present on machine
Normal Created 117s kubelet, node3 Created container nginx
Normal Started 117s kubelet, node3 Started container nginx
Error from server (Forbidden): Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy)
$ kubectl create clusterrolebinding system:anonymous --clusterrole=cluster-admin --user=system:anonymous