基于K3S构建高可用Rancher集群

192.168.111.21 nginx-master
192.168.111.22 nginx-backup
192.168.111.50 k3s-node1
192.168.111.51 k3s-node2
192.168.111.52 k3s-mysql

# 下载安装包
wget https://www.keepalived.org/software/keepalived-2.0.20.tar.gz
# 解压安装包
tar zxvf keepalived-2.0.20.tar.gz
# 编译安装keepalived
cd keepalived-2.0.20
mkdir /usr/local/keepalived
./configure --prefix=/usr/local/keepalived/
make && make install
# 配置 keepalived 为系统服务
cp /usr/local/keepalived/sbin/keepalived /usr/sbin/keepalived
cp /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig/keepalived
touch /etc/init.d/keepalived
chmod +x /etc/init.d/keepalived # keepalived 中的内容见下文
vim /etc/init.d/keepalived
# 配置 keepalived
mkdir /etc/keepalived/
cp /usr/local/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/
vim /etc/keepalived/keepalived.conf #keepalived.conf 中的内容见下文
# 启动keepalived
systemctl start keepalived
systemctl enable keepalived
# 验证
systemctl status keepalived
# 此时keepalived应该是运行，一个为master，一个为backup， master上执行 ip addr 命令时，应该存在一个虚拟ip地址，backup上不应该有
# 访问 https://192.168.111.20 验证配置
# /etc/init.d/keepalived文件内容
#!/bin/sh
#
# Startup script for the Keepalived daemon
#
# processname: keepalived
# pidfile: /var/run/keepalived.pid
# config: /etc/keepalived/keepalived.conf
# chkconfig: - 21 79
# description: Start and stop Keepalived

# Source function library
. /etc/rc.d/init.d/functions

# Source configuration file (we set KEEPALIVED_OPTIONS there)
. /etc/sysconfig/keepalived

RETVAL=0

prog="keepalived"

start() {
    echo -n $"Starting $prog: "
    daemon keepalived ${KEEPALIVED_OPTIONS}
    RETVAL=$?
    echo
    [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog
}

stop() {
    echo -n $"Stopping $prog: "
    killproc keepalived
    RETVAL=$?
    echo
    [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog
}

reload() {
    echo -n $"Reloading $prog: "
    killproc keepalived -1
    RETVAL=$?
    echo
}

# See how we were called.
case "$1" in
    start)
        start
        ;;
    stop)
        stop
        ;;
    reload)
        reload
        ;;
    restart)
        stop
        start
        ;;
    condrestart)
        if [ -f /var/lock/subsys/$prog ]; then
            stop
            start
        fi
        ;;
    status)
        status keepalived
        RETVAL=$?
        ;;
    *)
        echo "Usage: $0 {start|stop|reload|restart|condrestart|status}"
        RETVAL=1
esac

exit $RETVAL
# /etc/keepalived/keepalived.conf 中的内容
! Configuration File for keepalived

global_defs {
   router_id 192.168.111.21 # 此id在网络中有且只有一个，不应有重复的id
}

vrrp_script chk_nginx {     #因为要检测nginx服务状态，所以创建一个检查脚本
    script "/usr/local/keepalived/check_ng.sh"
    interval 3
}

vrrp_instance VI_1 {
    state MASTER    # 配置此节点为master，备机上设置为BACKUP
    interface ens33    # 设置绑定的网卡
    virtual_router_id 51    # vrrp 组， 主备的vrrp组应该一样
    priority 120    # 优先级，优先级大的为主
    advert_int 1    # 检查间隔
    authentication { # 认证
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress { # 虚拟IP
        192.168.111.20
    }
    track_script {    # 执行脚本
        chk_nginx
    }
}
# /usr/local/keepalived/check_ng.sh 中的内容
#!/bin/bash
d=`date --date today +%Y%m%d_%H:%M:%S`
n=`ps -C nginx --no-heading|wc -l`
if [ $n -eq "0" ]; then
        nginx
        n2=`ps -C nginx --no-heading|wc -l`
        if [ $n2 -eq "0"  ]; then
                echo "$d nginx down,keepalived will stop" >> /var/log/check_ng.log
                systemctl stop keepalived
        fi
fi

# 移除旧版本Docker
sudo apt-get remove docker docker-engine docker.io containerd runc
# 安装工具包
sudo apt-get install -y \
    apt-transport-https \
    ca-certificates \
    curl \
    gnupg-agent \
    software-properties-common
# 添加 Docker官方 GPG key
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
# 添加 stable apt 源
sudo add-apt-repository \
   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
   $(lsb_release -cs) \
   stable"
# 安装 Docker-ce
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
# 验证安装
docker info
# 将当前用户加入"docker"用户组，加入到该用户组的账号在随后安装过程会用到。用于节点访问的SSH用户必须是节点上docker组的成员
sudo usermod -aG docker $USER

# 更新nginx配置文件
# vim /usr/local/nginx/conf/nginx.conf

#user  nobody;
worker_processes  4;
worker_rlimit_nofile 40000;

events {
    worker_connections  8192;
}

stream {
    upstream rancher_servers_http {
        least_conn;
        server 192.168.111.50:80 max_fails=3 fail_timeout=5s;
        server 192.168.111.51:80 max_fails=3 fail_timeout=5s;
    }
    server {
        listen     80;
        proxy_pass rancher_servers_http;
    }

    upstream rancher_servers_https {
        least_conn;
        server 192.168.111.50:443 max_fails=3 fail_timeout=5s;
        server 192.168.111.51:443 max_fails=3 fail_timeout=5s;
    }

    server {
        listen     443;
        proxy_pass rancher_servers_https;
    }
}

# 下载地址：https://dev.mysql.com/downloads/mysql/5.7.html#downloads
# 创建运行MySQL数据库的用户和用户组
groupadd -r mysql
useradd -r -g mysql mysql
# 解压安装包，更改目录权限，创建数据库目录
tar zxvf mysql-5.7.30-linux-glibc2.12-x86_64.tar.gz
mkdir -p /app/mysql/data
mv mysql-5.7.30-linux-glibc2.12-x86_64/* /app/mysql/
chown -R mysql:mysql /app/mysql
# 初始化数据库
cd /app/mysql
./bin/mysqld --initialize \
--user=mysql --basedir=/app/mysql/ \
--datadir=/app/mysql/data/
# !!注意最后一行的初始密码
7Jlhi:gg?rE0
# 创建RSA private key
./bin/mysql_ssl_rsa_setup --datadir=/app/mysql/data/
# 添加 MySQL 到开机启动，修改/etc/init.d/mysqld中的basedir和datadir
cp support-files/mysql.server /etc/init.d/mysqld

basedir=/app/mysql
datadir=/app/mysql/data

chkconfig mysqld on
# 修改环境变量
vim /etc/profile
# 添加
export PATH=/app/mysql/bin:$PATH
# 使环境变量生效
source /etc/profile

# 备份系统自带的/etc/my.cnf，在/app/mysql/目录新建my.cnf,并且将文件属性调整为mysql:mysql
mv /etc/my.cnf /etc/my.cnf.bak
touch /app/mysql/my.cnf    # 具体内容见下文

# 启动mysql
/etc/init.d/mysqld start
# 创建mysql.sock软链接
ln -s /app/mysql/mysql.sock /tmp/mysql.sock
# 使用初始密码登陆
mysql -uroot -p

# 登陆成功后修改密码
alter user 'root'@'localhost' identified by "12345678";
flush privileges;

# 配置数据库远程登录
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '12345678' WITH GRANT OPTION;
flush privileges;
# 验证 略

# my.cnf
[mysqld]
character-set-server=utf8
datadir=/app/mysql/data
socket=/app/mysql/mysql.sock
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0
# Settings user and group are ignored when systemd is used.
# If you need to run mysqld under a different user or group,
# customize your systemd unit file for mariadb according to the
# instructions in http://fedoraproject.org/wiki/Systemd

#
# include all files from the config directory
#
!includedir /etc/my.cnf.d

[client]
character-set-server=utf8
socket=/app/mysql/mysql.sock

[mysql]
character-set-server=utf8
socket=/app/mysql/mysql.sock

# 启动 k3s Server
# ！注意，所有k3s节点上都要运行此命令
curl -sfL https://docs.rancher.cn/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn sh -s - server \
--datastore-endpoint="mysql://root:12345678@tcp(192.168.111.52:3306)/k3s"
# 验证
sudo k3s kubectl get nodes

# 在每个 Rancher Server 节点上安装 K3s 时，会在节点上/etc/rancher/k3s/k3s.yaml位置创建一个kubeconfig文件。该文件包含用于完全访问集群的凭据。# 复制 k3s.yaml 到 ~/.kube/config
sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
# 验证 kubectl
sudo kubectl get pods --all-namespaces

kube-system   coredns-8655855d6-c26h8                  1/1     Running     0          11m
kube-system   metrics-server-7566d596c8-v65fd          1/1     Running     0          11m
kube-system   helm-install-traefik-ttrfg               0/1     Completed   0          11m
kube-system   svclb-traefik-hxmzw                      2/2     Running     0          8m16s
kube-system   svclb-traefik-zxmg2                      2/2     Running     0          8m16s
kube-system   traefik-758cd5fc85-xsxbm                 1/1     Running     0          8m16s
kube-system   local-path-provisioner-6d59f47c7-497rl   1/1     Running     0          11m