ubuntu 16.04部署kubernetes集群【详细教程】

# 关闭防火墙ufw disable && ufw status# 执行脚本安装dockercurl -s  https://raw.githubusercontent.com/jy1779/docker/master/install/aliyun_docker_install.sh | bash# 修改docker.server参数LINE=$(grep -n ExecStart /lib/systemd/system/docker.service|awk -F : '{print $1}')
EXECSTARTPOST='ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT'
sed "$LINE a$EXECSTARTPOST" -i /lib/systemd/system/docker.service# 重新加载docker.server及重启docker服务systemctl daemon-reload && service docker restart

# 添加内核参数# 参数说明：# Controls IP packet forwardingnet.ipv4.ip_forward = 1# Enable netfilter on bridges.net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1

cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF# 使内核参数生效sysctl -p /etc/sysctl.d/k8s.conf# 如提示以下报错，则执行：modprobe br_netfiltersysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-ip6tables: No such file or directory
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: No such file or directory# 添加hosts文件echo -e "192.168.1.72 master\n192.168.1.73 node01\n192.168.1.74 node02" >> /etc/hosts# 获取k8s二进制文件及配置文件# kubernetes.git 并非官网的文件，是自定义安装k8s集群所需的文件root@master:~# git clone https://code.aliyun.com/jy1779/kubernetes.git# 解压k8s二进制文件，并添加到系统环境变量root@master:~# tar xf ./kubernetes/kubernetes-bins.tar.gz -C /usr/local/sbin/ && rm -f ./kubernetes/kubernetes-bins.tar.gz
root@master:~# echo 'export PATH=$PATH:/usr/local/sbin/kubernetes-bins' >> /etc/profile && source /etc/profile# 检测环境变量root@master:~# which kubectl 
/usr/local/sbin/kubernetes-bins/kubectl# 生成配置文件cd /root/kubernetes/kubernetes-starter/#修改配置文件vim config.properties#kubernetes二进制文件目录,eg: /home/michael/binBIN_PATH=/usr/local/sbin/kubernetes-bins#当前节点ip, eg: 192.168.1.102NODE_IP=192.168.1.72#etcd服务集群列表, eg: http://192.168.1.102:2379#如果已有etcd集群可以填写现有的。没有的话填写：http://${MASTER_IP}:2379 （MASTER_IP自行替换成自己的主节点ip）ETCD_ENDPOINTS=https://192.168.1.72:2379,https://192.168.1.73:2379,https://192.168.1.74:2379#kubernetes主节点ip地址, eg: 192.168.1.102MASTER_IP=192.168.1.72
root@master:~/kubernetes/kubernetes-starter# ./gen-config.sh with-ca
====替换变量列表====
BIN_PATH=/usr/local/sbin/kubernetes-bins
NODE_IP=192.168.1.72
ETCD_ENDPOINTS=https://192.168.1.72:2379,https://192.168.1.73:2379,https://192.168.1.74:2379
MASTER_IP=192.168.1.72
====================
====替换配置文件====
all-node/kube-calico.service
ca/admin/admin-csr.json
ca/ca-config.json
ca/ca-csr.json
ca/calico/calico-csr.json
ca/etcd/etcd-csr.json
ca/kube-proxy/kube-proxy-csr.json
ca/kubernetes/kubernetes-csr.json
master-node/etcd.service
master-node/kube-apiserver.service
master-node/kube-controller-manager.service
master-node/kube-scheduler.service
services/kube-dashboard.yaml
services/kube-dns.yaml
worker-node/10-calico.conf
worker-node/kubelet.service
worker-node/kube-proxy.service
=================配置生成成功，位置: /root/kubernetes/kubernetes-starter/target# 安装cfsslwget -q --show-progress --https-only --timestamping \
  https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \
  https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 #修改为可执行权限chmod +x cfssl_linux-amd64 cfssljson_linux-amd64#移动到bin目录mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson#验证cfssl version# 生成根证书# 创建目录存放ca证书mkdir -p /etc/kubernetes/ca# 提示：ca-config.json、ca-csr.json事先已经准备好，可修改，也可以自己生成# 复制ca文件cp ~/kubernetes/kubernetes-starter/target/ca/ca-config.json /etc/kubernetes/ca
cp ~/kubernetes/kubernetes-starter/target/ca/ca-csr.json /etc/kubernetes/ca# 生成证书和密钥cd /etc/kubernetes/ca
cfssl gencert -initca ca-csr.json | cfssljson -bare ca# 查看证书和密钥root@master:/etc/kubernetes/ca# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem

#创建存放etcd证书的目录mkdir -p /etc/kubernetes/ca/etcd#复制etcd证书配置cp ~/kubernetes/kubernetes-starter/target/ca/etcd/etcd-csr.json /etc/kubernetes/ca/etcd/
cd /etc/kubernetes/ca/etcd/#使用根证书(ca.pem)签发etcd证书cfssl gencert \
        -ca=/etc/kubernetes/ca/ca.pem \
        -ca-key=/etc/kubernetes/ca/ca-key.pem \
        -config=/etc/kubernetes/ca/ca-config.json \
        -profile=kubernetes etcd-csr.json | cfssljson -bare etcd#跟之前类似生成三个文件etcd.csr是个中间证书请求文件，我们最终要的是etcd-key.pem和etcd.pemls
etcd.csr  etcd-csr.json  etcd-key.pem  etcd.pem# 启动etcd服务#创建工作目录(保存数据的地方)mkdir -p /var/lib/etcd#把etcd服务配置文件copy到系统服务目录cp ~/kubernetes/kubernetes-starter/target/master-node/etcd.service /lib/systemd/system/# 创建etcd服务systemctl enable etcd.service#启动etcd服务service etcd start# 查看服务日志，看是否有错误信息，确保服务正常journalctl -f -u etcd.service#测试etcd服务是否正常ETCDCTL_API=3 etcdctl \
  --endpoints=https://192.168.1.72:2379  \
  --cacert=/etc/kubernetes/ca/ca.pem \
  --cert=/etc/kubernetes/ca/etcd/etcd.pem \
  --key=/etc/kubernetes/ca/etcd/etcd-key.pem \
  endpoint health#显示以下则为部署成功。https://192.168.1.72:2379 is healthy: successfully committed proposal: took = 10.408412ms

# 创建存放api证书目录mkdir -p /etc/kubernetes/ca/kubernetes#复制apiserver证书配置cp ~/kubernetes/kubernetes-starter/target/ca/kubernetes/kubernetes-csr.json /etc/kubernetes/ca/kubernetes/#使用根证书(ca.pem)签发kubernetes证书cd /etc/kubernetes/ca/kubernetes/
cfssl gencert \
        -ca=/etc/kubernetes/ca/ca.pem \
        -ca-key=/etc/kubernetes/ca/ca-key.pem \
        -config=/etc/kubernetes/ca/ca-config.json \
        -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes#跟之前类似生成三个文件kubernetes.csr是个中间证书请求文件，我们最终要的是kubernetes-key.pem和kubernetes.pemls
kubernetes.csr  kubernetes-csr.json  kubernetes-key.pem  kubernetes.pem#生成token认证文件#生成随机tokenhead -c 16 /dev/urandom | od -An -t x | tr -d ' '

root@master:/etc/kubernetes/ca/kubernetes# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
97e8c07dce2b2bab69cfd3162d5383c9# 写入token.csv文件echo "97e8c07dce2b2bab69cfd3162d5383c9,kubelet-bootstrap,10001,"system:kubelet-bootstrap"" > /etc/kubernetes/ca/kubernetes/token.csv# 把apiservice服务配置文件copy到系统服务目录cp ~/kubernetes/kubernetes-starter/target/master-node/kube-apiserver.service /lib/systemd/system/# 创建kube-apiserver服务systemctl enable kube-apiserver.service# 启动kube-apiserver服务service kube-apiserver start# 查看kube-apiserver日志journalctl -f -u kube-apiserver

# 把kube-controller-manager.service 服务配置文件copy到系统服务目录cp ~/kubernetes/kubernetes-starter/target/master-node/kube-controller-manager.service /lib/systemd/system/# 创建kube-controller-manager.service 服务systemctl enable kube-controller-manager.service# 启动kube-controller-manager.service 服务service kube-controller-manager start# 查看kube-controller-manager.service 日志journalctl -f -u kube-controller-manager

# 把scheduler 服务配置文件copy到系统服务目录cp ~/kubernetes/kubernetes-starter/target/master-node/kube-scheduler.service /lib/systemd/system/# 创建kube-scheduler.service 服务systemctl enable kube-scheduler.service# 启动kube-scheduler.service 服务service kube-scheduler start# 查看kube-scheduler.service 日志journalctl -f -u kube-scheduler

# 创建存放kubectl证书目录mkdir -p /etc/kubernetes/ca/admin# 准备admin证书配置 - kubectl只需客户端证书，因此证书请求中 hosts 字段可以为空# 复制kubectl证书配置cp ~/kubernetes/kubernetes-starter/target/ca/admin/admin-csr.json /etc/kubernetes/ca/admin/# 使用根证书(ca.pem)签发admin证书cd /etc/kubernetes/ca/admin/
cfssl gencert \
        -ca=/etc/kubernetes/ca/ca.pem \
        -ca-key=/etc/kubernetes/ca/ca-key.pem \
        -config=/etc/kubernetes/ca/ca-config.json \
        -profile=kubernetes admin-csr.json | cfssljson -bare admin# 我们最终要的是admin-key.pem和admin.pemls
admin.csr  admin-csr.json  admin-key.pem  admin.pem# 配置kubectl文件# 指定apiserver的地址和证书位置kubectl config set-cluster kubernetes \
        --certificate-authority=/etc/kubernetes/ca/ca.pem \
        --embed-certs=true \
        --server=https://192.168.1.72:6443# 设置客户端认证参数，指定admin证书和秘钥kubectl config set-credentials admin \
        --client-certificate=/etc/kubernetes/ca/admin/admin.pem \
        --embed-certs=true \
        --client-key=/etc/kubernetes/ca/admin/admin-key.pem# 关联用户和集群kubectl config set-context kubernetes \
        --cluster=kubernetes --user=admin# 设置当前上下文kubectl config use-context kubernetes# 设置结果就是一个配置文件，可以看看内容cat ~/.kube/config# 验证master节点,etcd-1和etcd-2还没有部署root@master:/etc/kubernetes/ca/admin# kubectl get componentstatus
NAME                 STATUS      MESSAGE                                                                                            ERROR
etcd-2               Unhealthy   Get https://192.168.1.74:2379/health: dial tcp 192.168.1.74:2379: getsockopt: connection refused   
scheduler            Healthy     ok                                                                                                 
etcd-1               Unhealthy   Get https://192.168.1.73:2379/health: dial tcp 192.168.1.73:2379: getsockopt: connection refused   
controller-manager   Healthy     ok                                                                                                 
etcd-0               Healthy     {"health": "true"} 
# 创建kubelet-bootstrap绑定kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

# calico证书用在四个地方：# calico/node: 这个docker 容器运行时访问 etcd 使用证书# cni 配置文件中 cni 插件: 需要访问 etcd 使用证书# calicoctl: 操作集群网络时访问 etcd 使用证书# calico/kube-controllers: 同步集群网络策略时访问 etcd 使用证书#创建存放calico证书mkdir -p /etc/kubernetes/ca/calico#准备calico证书配置 - calico只需客户端证书，因此证书请求中 hosts 字段可以为空cp ~/kubernetes/kubernetes-starter/target/ca/calico/calico-csr.json /etc/kubernetes/ca/calico/
cd /etc/kubernetes/ca/calico/
cfssl gencert \
        -ca=/etc/kubernetes/ca/ca.pem \
        -ca-key=/etc/kubernetes/ca/ca-key.pem \
        -config=/etc/kubernetes/ca/ca-config.json \
        -profile=kubernetes calico-csr.json | cfssljson -bare calico#我们最终要的是calico-key.pem和calico.pemls 
calico.csr  calico-csr.json  calico-key.pem  calico.pem#启动kube-calico.service 服务cp ~/kubernetes/kubernetes-starter/target/all-node/kube-calico.service /lib/systemd/system/
systemctl enable kube-calico.service
service kube-calico start
journalctl -f -u kube-calico#查看节点情况calicoctl node status