Kubernetes展望与思考之1.17初体验

接触kubernetes两年有余，从18年初加入kubernetes社区来算，已经一年半，或许是时候写点什么。简单文章如何写好，难点文章如何写透，或许是一种学问，打算推一个系列：《Kubernetes GO》算是对这两年的一个总结。

# $ docker pull k8s.gcr.io/kube-cross:v1.12.9-1
# k8s.gcr.io可以转换为gcr.io/google_containers
$ docker pull gcr.azk8s.cn/google_containers/kube-cross:v1.12.9-1
$ docker tag gcr.azk8s.cn/google_containers/kube-cross:v1.12.9-1 k8s.gcr.io/kube-cross:v1.12.9-1

tar -zxvf prepare.tgz
# docker load -i 解压的tar包，最终得到下列镜像
k8s.gcr.io/kube-cross                       v1.12.9-1   a808db72440c    5 weeks ago         1.87GB
k8s.gcr.io/debian-iptables-amd64            v11.0.2     01a746008995    5 months ago        45.4MB
k8s.gcr.io/debian-base-amd64                v1.0.0      204e96332c91    5 months ago        42.3MB
k8s.gcr.io/debian-hyperkube-base-amd64      v0.12.1     a46476511725    7 months ago        393MB

/root/go/src/k8s.io/kubernetes/build/lib/release.sh:行385: docker_build_opts[@]: 为绑定变量
/root/go/src/k8s.io/kubernetes/build/lib/release.sh:行385: docker_build_opts[@]: 为绑定变量
/root/go/src/k8s.io/kubernetes/build/lib/release.sh:行385: docker_build_opts[@]: 为绑定变量
/root/go/src/k8s.io/kubernetes/build/lib/release.sh:行385: docker_build_opts[@]: 为绑定变量

git diff build/lib/release.sh
diff --git a/build/lib/release.sh b/build/lib/release.sh
index 73c0bcc..e7bd1b1 100644
--- a/build/lib/release.sh
+++ b/build/lib/release.sh
@@ -382,7 +382,7 @@ EOF
         if [[ "${KUBE_BUILD_PULL_LATEST_IMAGES}" =~ [yY] ]]; then
             docker_build_opts+=("--pull")
         fi
-        "${DOCKER[@]}" build "${docker_build_opts[@]}" -q -t "${docker_image_tag}" "${docker_build_path}" >/dev/null
+        "${DOCKER[@]}" build -q -t "${docker_image_tag}" "${docker_build_path}" >/dev/null

ls _output/release-images/amd64/
conformance-amd64.tar  kube-apiserver.tar           kube-proxy.tar
hyperkube-amd64.tar    kube-controller-manager.tar  kube-scheduler.tar

ls _output/dockerized/bin/linux/amd64/
apiextensions-apiserver  e2e.test            genyaml     hyperkube                kubelet         mounter
conversion-gen           gendocs             ginkgo      kubeadm                  kubemark        openapi-gen
deepcopy-gen             genkubedocs         go2make     kube-apiserver           kube-proxy
defaulter-gen            genman              go-bindata  kube-controller-manager  kube-scheduler
e2e_node.test            genswaggertypedocs  go-runner   kubectl                  linkcheck
# 将kubelet、kubeadm、kubectl拷贝到/usr/bin/目录。

// 输入代码内容
cat /etc/hosts
10.142.21.132 kargo
10.142.114.189 paasn1

systemctl stop firewalld
systemctl disable firewalld

# 临时禁用selinux
setenforce 0
# 永久禁用selinux
vi /etc/selinux/config
SELINUX=disabled

cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl -p /etc/sysctl.d/k8s.conf

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip\_vs
modprobe -- ip\_vs\_rr
modprobe -- ip\_vs\_wrr
modprobe -- ip\_vs\_sh
modprobe -- nf\_conntrack\_ipv4
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip\_vs -e nf\_conntrack\_ipv4

# 临时禁用swap即可，重启后，需要再此执行，当然也可以把磁盘信息写入/etc/fstab
swapoff -a

yum install -y yum-utils \
  device-mapper-persistent-data \
  lvm2
yum-config-manager \
--add-repo \
https://download.docker.com/linux/centos/docker-ce.repo

# 18.09.7
yum install docker-ce docker-ce-cli containerd.io -y
# 通过yum list docker-ce --showduplicates | sort -r 获取版本信息，执行以下命令安装指定版本
# yum install docker-ce-<VERSION\_STRING> docker-ce-cli-<VERSION\_STRING> containerd.i

{

  "exec-opts": "native.cgroupdriver=systemd"

}

systemctl restart docker

docker info | grep Cgroup
Cgroup Driver: systemd

systemctl enable docker

cat <<EOF >/etc/systemd/system/kubelet.service.d/10-kubeadm.conf
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET\_KUBECONFIG\_ARGS=--v=6 --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET\_CONFIG\_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET\_KUBEADM\_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET\_EXTRA\_ARGS should be sourced from this file.
EnvironmentFile=-/etc/sysconfig/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET\_KUBECONFIG\_ARGS $KUBELET\_CONFIG\_ARGS $KUBELET\_KUBEADM\_ARGS $KUBELET\_EXTRA\_ARGS
EOF
cat <<EOF >/etc/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=[http://kubernetes.io/docs/](http://kubernetes.io/docs/)
[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF

# enable kubelet服务
systemctl daemon-reloads
systemctl enable kubelet.service

apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:undefinedapiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.16.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
scheduler: {}

其中镜像地址可通过imageRepository参数进行自定义，若本地已加载所需镜像，则无须改变，根据测试环境的实际情况,kubernetesVersion对应`kubeadm version`中GitVersion版本，将kubeadm.conf修改为：

cat <<EOF >/root/kubeadm.conf
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.142.21.132
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: kargo
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.17.0-alpha.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
scheduler: {}
EOF

接下来使用kubeadm初始化集群，选择kargo作为Master Node，在kargo上执行下面的命令：

该命令会生成以下内容：访问集群`mkdir -p $HOME/.kube

关于kubernetes的接入说明，具体可参看[集群接入](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/)验证集群查看一下集群状态，确认个组件都处于healthy状态：

kubectl cluster-info
Kubernetes master is running at https://10.233.0.1
CoreDNS is running at https://10.233.0.1/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
    - system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
    - signing
    - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 1.2.3.4
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: kargo
  taints:
    - effect: NoSchedule
key: node-role.kubernetes.io/master

# kubectl get cs
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok
scheduler            Healthy   ok
etcd-0               Healthy   {"health":"true"}

# 生成token
kubeadm token create
a9trii.8d52xbbusol0glji

openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.\* //'
31b110e7fea9c86ddc6c1caa9f27f39021ea115ab7d2b5a86e157c42d9c8c57c

kubeadm join --discovery-token 9trii.8d52xbbusol0glji --discovery-token-ca-cert-hash sha256:31b110e7fea9c86ddc6c1caa9f27f39021ea115ab7d2b5a86e157c42d9c8c57c 10.142.21.132:6443 -v=6

kubectl get node
NAME    STATUS   ROLES    AGE   VERSION
kargo   Ready    master   57m   v1.17.0-alpha.0
passn1   Ready    <none>   11s   v1.17.0-alpha.0

docker ps|grep etcd
0214f9a78ba5        b2756210eeab                              "etcd --advertise-cl…"   4 hours ago         Up 4 hours                              k8s\_etcd\_etcd-kargo\_kube-system\_2b66f634d9a00ad56540109b231dd318\_3
20419a0de748        k8s.gcr.io/pause:3.1                      "/pause"                 9 days ago          Up 1 days                               k8s\_POD\_etcd-kargo\_kube-system\_2b66f634d9a00ad56540109b231dd318\_2

 docker exec -it 168729f100e0 etcdctl --endpoint https://127.0.0.1:2379   \
 --endpoint https://127.0.0.1:2379    \
 --ca-file=/etc/ssl/etcd/ssl/ca.pem \
 --cert-file=/etc/ssl/etcd/ssl/ca.pem \
 --key-file=/etc/ssl/etcd/ssl/ca-key.pem \
 cluster-health
 member 38ee253c41f760ca is healthy: got healthy result from https://10.142.21.132:2379
 cluster is healthy

kubeadm reset
rm -rf /var/lib/cni/ /var/lib/etcd/
rm -rf /var/lib/kubelet /etc/kubernetes/

POD\_CIDR="<your-pod-cidr>" \
sed -i -e "s?192.168.0.0/16?$POD\_CIDR?g" calico.yaml

#kubectl get po -n kube-system|grep calico
calico-kube-controllers-744795b577-jc4r7                          1/1     Running                 0          1d
calico-node-7lbdm                                                 1/1     Running                 8          1d
#kubectl get po -n kube-system|grep dns
coredns-7f547f9899-7l9cq                                          1/1     Running                 0          1d
dns-autoscaler-7bf66d8bd8-l52bn                                   1/1     Running                 0          1d
kubectl run  busybox --image=busybox:latest --restart=Never

kubectl run -it busybox sh
nslookup kubernetes.default

Server:    10.233.0.3
Address 1: 10.233.0.3 kube-dns.kube-system.svc.cluster.local
Name:      kubernetes.default
Address 1: 10.233.0.3 kubernetes.default.svc.cluster.local
ping 10.142.21.132

若ping通，则网络正常

curl -O [https://get.helm.sh/helm-v2.14.3-linux-amd64.tar.gz](https://get.helm.sh/helm-v2.14.3-linux-amd64.tar.gz)
tar -zxvf helm-v2.14.3-linux-amd64.tar.gz
cd linux-amd64/
cp helm /usr/local/bin/

cat <<EOF >/root/helm.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: tiller
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: tiller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
    name: tiller
    namespace: kube-system
EOF

kubectl apply -f /root/helm.yaml

kubectl get pod -n kube-system -l app=helm
NAME                             READY   STATUS    RESTARTS   AGE
tiller-deploy-7d666f8ccc-92h27   1/1     Running   5          1d

helm repo add stable [http://mirror.azure.cn/kubernetes/charts](http://mirror.azure.cn/kubernetes/charts)
"stable" has been added to your repositories

helm repo list

NAME  URL
stable [http://mirror.azure.cn/kubernetes/charts](http://mirror.azure.cn/kubernetes/charts)
local http://127.0.0.1:8879/charts

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: metrics-server
  namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: metrics-server
  namespace: kube-system
  labels:
    k8s-app: metrics-server
spec:
  selector:
    matchLabels:
      k8s-app: metrics-server
  template:
    metadata:
      name: metrics-server
      labels:
        k8s-app: metrics-server
    spec:
      serviceAccountName: metrics-server
      volumes:
      # mount in tmp so we can safely use from-scratch images and/or read-only containers
      - name: tmp-dir
        emptyDir: {}
      containers:
      - name: metrics-server
        image: k8s.gcr.io/metrics-server-amd64:v0.3.4
        imagePullPolicy: Always
        args:
        - --logtostderr
        - --kubelet-insecure-tls
        - --kubelet-preferred-address-types=InternalIP
        volumeMounts:
        - name: tmp-dir
          mountPath: /tmp

kubectl get po -n kube-system -l k8s-app=metrics-server
NAME                              READY   STATUS    RESTARTS   AGE
metrics-server-66fc8cddfb-cwvcf   1/1     Running   6          1d

kubectl top node
NAME     CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%
kargo   1533m        19%    9430Mi          61%
paasn1   4443m        56%    10490Mi         67%

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 443
      targetPort: 8443
  selector:
    k8s-app: kubernetes-dashboard
  type: NodePort

kubectl get svc -n kubernetes-dashboard
NAME                   TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)         AGE
kubernetes-dashboard   NodePort   10.233.41.73   <none>        443:30479/TCP   1d获得主机端口30479，dashboard的界面绕过https,通过https://10.142.21.132:30479nodeport方式进行访问外，还可以执行以下代理操作kubectl proxy --accept-hosts='^.*' --address='10.142.113.20' --port 8080