$ wget https://dl.google.com/go/go1.13.8.linux-amd64.tar.gz
$ tar -zxvf go1.13.8.linux-amd64.tar.gz -C /usr/local/
export GOROOT=/usr/local/go
# GOPATH
export GOPATH=$HOME/go
# GOROOT bin
export PATH=$PATH:$GOROOT/bin
# GOPATH bin
export PATH=$PATH:$GOPATH/bin
$ source ~/.bashrc
$ git clone https://github.com/kubernetes/kubernetes.git
$ make KUBE_BUILD_PLATFORMS=linux/amd64
+++ [0215 22:16:44] Building go targets for linux/amd64:
./vendor/k8s.io/code-generator/cmd/deepcopy-gen
+++ [0215 22:16:52] Building go targets for linux/amd64:
./vendor/k8s.io/code-generator/cmd/defaulter-gen
+++ [0215 22:17:00] Building go targets for linux/amd64:
./vendor/k8s.io/code-generator/cmd/conversion-gen
+++ [0215 22:17:12] Building go targets for linux/amd64:
./vendor/k8s.io/kube-openapi/cmd/openapi-gen
+++ [0215 22:17:25] Building go targets for linux/amd64:
./vendor/github.com/go-bindata/go-bindata/go-bindata
+++ [0215 22:17:27] Building go targets for linux/amd64:
cmd/kube-proxy
cmd/kube-apiserver
cmd/kube-controller-manager
cmd/kubelet
cmd/kubeadm
cmd/kube-scheduler
vendor/k8s.io/apiextensions-apiserver
cluster/gce/gci/mounter
cmd/kubectl
cmd/gendocs
cmd/genkubedocs
cmd/genman
cmd/genyaml
cmd/genswaggertypedocs
cmd/linkcheck
vendor/github.com/onsi/ginkgo/ginkgo
test/e2e/e2e.test
cluster/images/conformance/go-runner
cmd/kubemark
vendor/github.com/onsi/ginkgo/ginkgo
$ pwd
/root/Coding/kubernetes/_output/local/bin/linux/amd64
$ ls
apiextensions-apiserver genman go-runner kube-scheduler kubemark
e2e.test genswaggertypedocs kube-apiserver kubeadm linkcheck
gendocs genyaml kube-controller-manager kubectl mounter
genkubedocs ginkgo kube-proxy kubelet
$ git clone https://github.com/wangao1236/k8s_cluster_deploy.git
$ cd k8s_cluster_deploy/scripts
$ chmod +x *.sh
$ cd ~
$ mkdir -p k8s/scripts
$ cp k8s_cluster_deploy/scripts/* k8s/scripts
$ cd k8s_cluster_deploy/http_scripts
$ chmod +x *.sh
$ cd ~
$ mkdir -p k8s/http_scripts
$ cp k8s_cluster_deploy/http_scripts/* k8s/http_scripts
$ cd k8s_cluster_deploy/https_scripts
$ chmod +x *.sh
$ cd ~
$ mkdir -p k8s/https_scripts
$ cp k8s_cluster_deploy/https_scripts/* k8s/http_scripts
$ curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
$ curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
$ curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
$ chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo
$ cat k8s_cluster_deploy/scripts/cfssl.sh
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo
$ mkdir -p /opt/etcd/{cfg,bin,ssl}
$ wget https://github.com/etcd-io/etcd/releases/download/v3.3.18/etcd-v3.3.18-linux-amd64.tar.gz
$ tar -zxvf etcd-v3.3.18-linux-amd64.tar.gz
$ cp etcd-v3.3.18-linux-amd64/etcdctl etcd-v3.3.18-linux-amd64/etcd /opt/etcd/bin
$ mkdir -p k8s/etcd-cert
$ cp k8s/scripts/etcd-cert.sh k8s/etcd-cert
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat > ca-csr.json <<EOF
{
"CN": "etcd CA",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing"
}
]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
#-----------------------
cat > server-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.1.65",
"192.168.1.66",
"192.168.1.67",
"192.168.1.68",
"192.168.1.69"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
$ ./etcd-cert.sh
2020/02/20 17:18:09 [INFO] generating a new CA key and certificate from CSR
2020/02/20 17:18:09 [INFO] generate received request
2020/02/20 17:18:09 [INFO] received CSR
2020/02/20 17:18:09 [INFO] generating key: rsa-2048
2020/02/20 17:18:09 [INFO] encoded CSR
2020/02/20 17:18:09 [INFO] signed certificate with serial number 712703952401219579947544408367305212876133158662
2020/02/20 17:18:09 [INFO] generate received request
2020/02/20 17:18:09 [INFO] received CSR
2020/02/20 17:18:09 [INFO] generating key: rsa-2048
2020/02/20 17:18:09 [INFO] encoded CSR
2020/02/20 17:18:09 [INFO] signed certificate with serial number 59975233056205858127163767550140095337822886214
2020/02/20 17:18:09 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
$ cp *.pem /opt/etcd/ssl
$ ./k8s/scripts/etcd.sh etcd01 192.168.1.67 etcd01=https://192.168.1.67:2380,etcd02=https://192.168.1.68:2380,etcd03=https://192.168.1.69:2380
#!/bin/bash
# example: ./etcd.sh etcd01 192.168.1.10 etcd01=https://192.168.1.10:2380,etcd02=https://192.168.1.11:2380,etcd03=https://192.168.1.12:2380
ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3
systemctl stop etcd
systemctl disable etcd
WORK_DIR=/opt/etcd
cat <<EOF >$WORK_DIR/cfg/etcd
#[Member]
ETCD_NAME="${ETCD_NAME}"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_LISTEN_CLIENT_URLS="https://${ETCD_IP}:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_IP}:2379"
ETCD_INITIAL_CLUSTER="${ETCD_CLUSTER}"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
cat <<EOF >/usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=${WORK_DIR}/cfg/etcd
ExecStart=${WORK_DIR}/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=${WORK_DIR}/ssl/server.pem \
--key-file=${WORK_DIR}/ssl/server-key.pem \
--peer-cert-file=${WORK_DIR}/ssl/server.pem \
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd
$ scp -r /opt/etcd/ root@192.168.1.68:/opt/
$ scp -r /opt/etcd/ root@192.168.1.69:/opt/
$ scp /usr/lib/systemd/system/etcd.service root@192.168.1.68:/usr/lib/systemd/system/
$ scp /usr/lib/systemd/system/etcd.service root@192.168.1.69:/usr/lib/systemd/system/
[root@192.168.1.68] $ vim /opt/etcd/cfg/etcd
#[Member]
ETCD_NAME="etcd02"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.68:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.68:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.68:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.68:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.1.67:2380,etcd02=https://192.168.1.68:2380,etcd03=https://192.168.1.69
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
[root@192.168.1.69] $ vim /opt/etcd/cfg/etcd
#[Member]
ETCD_NAME="etcd03"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.69:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.69:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.69:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.69:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.1.67:2380,etcd02=https://192.168.1.68:2380,etcd03=https://192.168.1.69
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
$ sudo systemctl enable etcd.service
Created symlink /etc/systemd/system/multi-user.target.wants/etcd.service → /usr/lib/systemd/system/etcd.service.
$ sudo systemctl start etcd.service
$ sudo etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379" cluster-health
member 3143a1397990e241 is healthy: got healthy result from https://192.168.1.68:2379
member 469e7b2757c25086 is healthy: got healthy result from https://192.168.1.67:2379
member 5b1e32d0ab5e3e1b is healthy: got healthy result from https://192.168.1.69:2379
cluster is healthy
$ /opt/etcd/bin/etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://127.0.0.1:2379" set /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}
$ /opt/etcd/bin/etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://127.0.0.1:2379" get /coreos.com/network/config
{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}
$ wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
$ tar -zxvf flannel-v0.11.0-linux-amd64.tar.gz
$ mkdir -p /opt/kubernetes/{cfg,bin,ssl}
$ mv mk-docker-opts.sh flanneld /opt/kubernetes/bin/
$ ./k8s/scripts/flannel.sh https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379
$ cat ./k8s/scripts/flannel.sh
#!/bin/bash
ETCD_ENDPOINTS=${1:-"http://127.0.0.1:2379"}
systemctl stop flanneld
systemctl disable flanneld
cat <<EOF >/opt/kubernetes/cfg/flanneld
FLANNEL_OPTIONS="--etcd-endpoints=${ETCD_ENDPOINTS} \\
-etcd-cafile=/opt/etcd/ssl/ca.pem \\
-etcd-certfile=/opt/etcd/ssl/server.pem \\
-etcd-keyfile=/opt/etcd/ssl/server-key.pem"
EOF
cat <<EOF >/usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker -f /run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable flanneld
systemctl restart flanneld
$ cat /run/flannel/subnet.envFLANNEL_NETWORK=172.17.0.0/16
FLANNEL_SUBNET=172.17.89.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true
$ cat /run/flannel/docker
DOCKER_OPT_BIP="--bip=172.17.89.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=false"
DOCKER_OPT_MTU="--mtu=1450"
DOCKER_OPTS=" --bip=172.17.89.1/24 --ip-masq=false --mtu=1450"
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
BindsTo=containerd.service
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=docker.socket
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
EnvironmentFile=/run/flannel/docker
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H unix:///var/run/docker.soc
#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.soc
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
......
$ systemctl daemon-reload
$ systemctl restart docker
$ ifconfig
docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 172.17.89.1 netmask 255.255.255.0 broadcast 172.17.89.255
ether 02:42:fb:16:3b:12 txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
enp0s3: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.0.2.15 netmask 255.255.255.0 broadcast 10.0.2.255
inet6 fe80::a00:27ff:feaf:b59f prefixlen 64 scopeid 0x20<link>
ether 08:00:27:af:b5:9f txqueuelen 1000 (Ethernet)
RX packets 517 bytes 247169 (247.1 KB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 361 bytes 44217 (44.2 KB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
enp0s8: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.1.67 netmask 255.255.255.0 broadcast 192.168.1.255
inet6 fe80::a00:27ff:fe9f:cb5c prefixlen 64 scopeid 0x20<link>
inet6 2409:8a10:2e24:d130:a00:27ff:fe9f:cb5c prefixlen 64 scopeid 0x0<global>
ether 08:00:27:9f:cb:5c txqueuelen 1000 (Ethernet)
RX packets 9244 bytes 2349434 (2.3 MB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 7420 bytes 1047863 (1.0 MB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1450
inet 172.17.89.0 netmask 255.255.255.255 broadcast 0.0.0.0
inet6 fe80::60c3:ecff:fe34:9d6c prefixlen 64 scopeid 0x20<link>
ether 62:c3:ec:34:9d:6c txqueuelen 0 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 6 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1000 (Local Loopback)
RX packets 3722 bytes 904859 (904.8 KB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 3722 bytes 904859 (904.8 KB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@adf9fc37d171 /]# yum install -y net-tools
[root@adf9fc37d171 /]# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 172.17.89.2 netmask 255.255.255.0 broadcast 172.17.89.255
ether 02:42:ac:11:59:02 txqueuelen 0 (Ethernet)
RX packets 1538 bytes 14149689 (13.4 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 1383 bytes 81403 (79.4 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
loop txqueuelen 1000 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@adf9fc37d171 /]# ping 172.17.89.1
PING 172.17.89.1 (172.17.89.1) 56(84) bytes of data.
64 bytes from 172.17.89.1: icmp_seq=1 ttl=64 time=0.045 ms
64 bytes from 172.17.89.1: icmp_seq=2 ttl=64 time=0.045 ms
64 bytes from 172.17.89.1: icmp_seq=3 ttl=64 time=0.050 ms
64 bytes from 172.17.89.1: icmp_seq=4 ttl=64 time=0.052 ms
64 bytes from 172.17.89.1: icmp_seq=5 ttl=64 time=0.049 ms
$ sudo apt-get -y install nginx
$ cat k8s_cluster_deploy/nginx/nginx.conf
user www-data;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
worker_processes 2;
worker_rlimit_nofile 65536;
events {
worker_connections 32768;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
log_format default '$remote_addr:$remote_port->$upstream_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
#mail {
# # See sample authentication script at:
# # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
#
# # auth_http localhost/auth.php;
# # pop3_capabilities "TOP" "USER";
# # imap_capabilities "IMAP4rev1" "UIDPLUS";
#
# server {
# listen localhost:110;
# protocol pop3;
# proxy on;
# }
#
# server {
# listen localhost:143;
# protocol imap;
# proxy on;
# }
#}
$ cat k8s_cluster_deploy/nginx/conf.d/k8s.conf
upstream kubernetes-api-cluster-tls {
server 192.168.1.67:6443 max_fails=0 fail_timeout=3s weight=1;
server 192.168.1.68:6443 max_fails=0 fail_timeout=3s weight=1;
# server 192.168.1.69:6443 weight=1 max_fails=0 fail_timeout=3s;
}
upstream kubernetes-api-cluster {
server 192.168.1.67:8080 weight=100 max_fails=0 fail_timeout=3s;
server 192.168.1.68:8080 weight=100 max_fails=0 fail_timeout=3s;
# server 192.168.1.69:8080 weight=100 max_fails=0 fail_timeout=3s;
}
server {
listen 8443 ssl;
ssl_certificate /etc/nginx/ssl/master/kube-apiserver.pem; # kube-apiserver cert
ssl_certificate_key /etc/nginx/ssl/master/kube-apiserver-key.pem; # kube-apiserver key
ssl_trusted_certificate /etc/nginx/ssl/ca.pem; # ca.pem
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS;
location / {
proxy_ssl_certificate /etc/nginx/ssl/test-user.pem; # kubectl cert
proxy_ssl_certificate_key /etc/nginx/ssl/test-user-key.pem; # kubectl key
proxy_ssl_trusted_certificate /etc/nginx/ssl/ca.pem; # ca.pem
proxy_pass https://kubernetes-api-cluster-tls;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504 http_403 http_404 http_429 non_idempotent;
proxy_next_upstream_timeout 1s;
proxy_next_upstream_tries 3;
proxy_set_header Host $host;
proxy_set_header X-Real-Ip $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-NginX-Proxy true;
proxy_read_timeout 600s;
}
access_log /var/log/nginx/access.log default;
}
server {
listen 8081;
location / {
proxy_pass http://kubernetes-api-cluster;
proxy_next_upstream error timeout http_500 http_502 http_503 http_504 http_403 http_429 non_idempotent;
proxy_next_upstream_timeout 3s;
proxy_next_upstream_tries 5;
proxy_ignore_client_abort on;
proxy_set_header Host $host;
proxy_set_header X-Real-Ip $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-NginX-Proxy true;
proxy_connect_timeout 300s;
}
#access_log /var/log/nginx/access.log default;
}
$ cd k8s/http_scripts
$ ./install.sh
$ cd k8s/https_scripts
$ ./install.sh
$ cat k8s/https_scripts/install.sh
#!/bin/bash
sudo mkdir -p /opt/kubernetes/{bin,cfg,log,ssl}
sudo rm -rf /opt/kubernetes/cfg/*
sudo rm -rf /opt/kubernetes/log/*
sudo rm -rf /opt/kubernetes/ssl/*
ssh root@master2 "mkdir -p /opt/kubernetes/{bin,cfg,log} && \
rm -rf /opt/kubernetes/cfg/* && \
rm -rf /opt/kubernetes/log/* && \
rm -rf /opt/kubernetes/ssl/*"
ssh root@master3 "mkdir -p /opt/kubernetes/{bin,cfg,log} && \
rm -rf /opt/kubernetes/cfg/* && \
rm -rf /opt/kubernetes/log/* && \
rm -rf /opt/kubernetes/ssl/*"
mkdir -p ../k8s-cert
sudo rm -rf ../k8s-cert/*
sudo rm -rf /opt/kubernetes/ssl/*
ssh root@master2 "rm -rf /opt/kubernetes/ssl/*"
ssh root@master3 "rm -rf /opt/kubernetes/ssl/*"
cp k8s-cert.sh ../k8s-cert
cd ../k8s-cert
./k8s-cert.sh
echo -e "\033[32m ======>>>>>>copy new cert \033[0m"
sudo cp -r ca* admin* test-user* master node /opt/kubernetes/ssl
sudo scp -r /opt/kubernetes/ssl root@master2:/opt/kubernetes/
sudo scp -r /opt/kubernetes/ssl root@master3:/opt/kubernetes/
cd ../https_scripts
mkdir -p ../config
sudo rm -rf ../config/*
sudo rm -rf /opt/kubernetes/cfg/*
ssh root@master2 "rm -rf /opt/kubernetes/cfg/*"
ssh root@master3 "rm -rf /opt/kubernetes/cfg/*"
cp config.sh ../config
cd ../config
sudo ./config.sh https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379 192.168.1.66 192.168.1.67 /opt/kubernetes/ssl
echo -e "\033[32m ======>>>>>>copy new config \033[0m"
sudo cp * /opt/kubernetes/cfg
sudo chown ao:ao config
sudo chown ao:ao test-user.config
cp config ~/.kube/
cp test-user.config ~/.kube/
sudo scp /opt/kubernetes/cfg/* root@master2:/opt/kubernetes/cfg/
scp config ao@master2:/home/ao/.kube/
sudo scp /opt/kubernetes/cfg/* root@master3:/opt/kubernetes/cfg/
scp config ao@master3:/home/ao/.kube/
cd ../https_scripts
echo -e "\033[32m ======>>>>>>restart nginx \033[0m"
ssh root@lb2 "systemctl stop nginx.service && \
systemctl disable nginx.service && \
rm /var/log/nginx/*"
sudo scp -r /opt/kubernetes/ssl/* root@lb2:/etc/nginx/ssl/
ssh root@lb2 "cd /etc/nginx/ssl && \
cat admin.pem > test.pem && \
cat admin-key.pem > test-key.pem && \
systemctl stop haproxy.service && \
systemctl daemon-reload && \
systemctl restart nginx.service && \
systemctl status nginx.service"
echo -e "\033[32m ======>>>>>>restart etcd \033[0m"
sudo systemctl stop etcd.service
ssh root@master2 "systemctl stop etcd.service"
ssh root@master3 "systemctl stop etcd.service"
sleep 5s
sudo rm -rf /var/lib/etcd/default.etcd/member
ssh root@master2 "rm -rf /var/lib/etcd/default.etcd/member"
ssh root@master3 "rm -rf /var/lib/etcd/default.etcd/member"
sleep 5s
sudo systemctl daemon-reload
sudo systemctl restart etcd.service
ssh root@master2 "systemctl daemon-reload && \
systemctl restart etcd.service && \
systemctl status etcd.service"
ssh root@master3 "systemctl daemon-reload && \
systemctl restart etcd.service && \
systemctl status etcd.service"
sudo etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.1.67:2379,https://192.168.1.68,https://192.168.1.69" cluster-health
sudo etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.1.67:2379,https://192.168.1.68,https://192.168.1.69" set /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
echo -e "\033[32m ======>>>>>>restart flannel && docker \033[0m"
sudo ./flannel.sh https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379
scp flannel.sh ao@master2:/home/ao/Coding/k8s/scripts && scp flannel.sh ao@master3:/home/ao/Coding/k8s/scripts
sudo systemctl daemon-reload
sudo systemctl restart docker
ssh root@master2 "hostname && \
cd /home/ao/Coding/k8s/scripts && \
./flannel.sh https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379 && \
systemctl daemon-reload && \
systemctl restart docker && \
systemctl status docker"
ssh root@master3 "hostname && \
cd /home/ao/Coding/k8s/scripts && \
./flannel.sh https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379 && \
systemctl daemon-reload && \
systemctl restart docker && \
systemctl status docker"
echo -e "\033[32m ======>>>>>>restart kube-apiserver \033[0m"
sudo systemctl stop kube-apiserver
ssh root@master2 systemctl stop kube-apiserver
ssh root@master3 systemctl stop kube-apiserver
sudo ./apiserver.sh 192.168.1.67 https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379
scp apiserver.sh ao@master2:/home/ao/Coding/k8s/https_scripts && scp apiserver.sh ao@master3:/home/ao/Coding/k8s/https_scripts
ssh root@master2 "hostname && \
cd /home/ao/Coding/k8s/https_scripts && \
./apiserver.sh 192.168.1.68 https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379"
ssh root@master3 "hostname && \
cd /home/ao/Coding/k8s/https_scripts && \
./apiserver.sh 192.168.1.69 https://192.168.1.67:2379,https://192.168.1.68:2379,https://192.168.1.69:2379"
echo -e "\033[32m ======>>>>>>restart kube-controller-manager \033[0m"
sudo systemctl stop kube-controller-manager
ssh root@master2 systemctl stop kube-controller-manager
ssh root@master3 systemctl stop kube-controller-manager
sudo ./controller-manager.sh
scp controller-manager.sh ao@master2:/home/ao/Coding/k8s/https_scripts && scp controller-manager.sh ao@master3:/home/ao/Coding/k8s/https_scripts
ssh root@master2 "hostname && \
cd /home/ao/Coding/k8s/https_scripts && \
./controller-manager.sh"
ssh root@master3 "hostname && \
cd /home/ao/Coding/k8s/https_scripts && \
./controller-manager.sh"
echo -e "\033[32m ======>>>>>>restart kube-scheduler \033[0m"
sudo systemctl stop kube-scheduler
ssh root@master2 systemctl stop kube-scheduler
ssh root@master3 systemctl stop kube-scheduler
sudo ./scheduler.sh
scp scheduler.sh ao@master2:/home/ao/Coding/k8s/https_scripts && scp scheduler.sh ao@master3:/home/ao/Coding/k8s/https_scripts
ssh root@master2 "hostname && \
cd /home/ao/Coding/k8s/https_scripts && \
./scheduler.sh"
ssh root@master3 "hostname && \
cd /home/ao/Coding/k8s/https_scripts && \
./scheduler.sh"
echo -e "\033[32m ======>>>>>>restart kubelet \033[0m"
sudo systemctl stop kubelet
ssh root@master2 systemctl stop kubelet
ssh root@master3 systemctl stop kubelet
sudo ./kubelet.sh 192.168.1.67 node1
scp kubelet.sh ao@master2:/home/ao/Coding/k8s/https_scripts && scp kubelet.sh ao@master3:/home/ao/Coding/k8s/https_scripts
ssh root@master2 "hostname && \
cd /home/ao/Coding/k8s/https_scripts && \
./kubelet.sh 192.168.1.68 node2"
ssh root@master3 "hostname && \
cd /home/ao/Coding/k8s/https_scripts && \
./kubelet.sh 192.168.1.69 node3"
echo -e "\033[32m ======>>>>>>restart proxy \033[0m"
sudo systemctl stop kube-proxy
ssh root@master2 systemctl stop kube-proxy
ssh root@master3 systemctl stop kube-proxy
sudo ./proxy.sh node1
scp proxy.sh ao@master2:/home/ao/Coding/k8s/https_scripts && scp proxy.sh ao@master3:/home/ao/Coding/k8s/https_scripts
ssh root@master2 "hostname && \
cd /home/ao/Coding/k8s/https_scripts && \
./proxy.sh node2"
ssh root@master3 "hostname && \
cd /home/ao/Coding/k8s/https_scripts && \
./proxy.sh node3"
kubectl delete -f ../yamls/jobs-tester.yaml
kubectl delete clusterrolebinding jobs-test
kubectl delete clusterrolebinding test-cluster-admin-binding
kubectl apply -f ../yamls/jobs-tester.yaml
kubectl create clusterrolebinding jobs-test --clusterrole=jobs-tester --user=test-user
kubectl create clusterrolebinding test-cluster-admin-binding --clusterrole=cluster-admin --user=test-user
echo "1st"
sleep 10s
kubectl label node node1 node2 node3 node-role.kubernetes.io/master=true
echo "2nd"
kubectl get nodes --all-namespaces
kubectl taint nodes --all node-role.kubernetes.io/master=true:NoSchedule
kubectl taint nodes --all node-role.kubernetes.io/master-
sleep 10s
echo "3rd"
kubectl get nodes --all-namespaces
kubectl delete -f ../yamls/nginx-deployment.yaml
sleep 5s
kubectl apply -f ../yamls/nginx-deployment.yaml
kubectl get pods --all-namespaces