kuberntes中的redis集群性能调优

# sysctl.conf中配置fs.file-max、net.core.somaxconn两个属性
$ cat << EOF >> /etc/sysctl.conf
fs.file-max=655350
net.core.somaxconn=20480
EOF
sysctl -p

# limits.conf中配置文件句柄数及进程数的硬限制和软限制
$ cat << 'EOF' >> /etc/security/limits.conf
*	hard	nofile	655350
*	soft	nofile	655350
*	hard	nproc	655350
*	soft	nproc	655350
EOF

# 关闭内存transparent_hugepage特性
$ cat << 'EOF' >> /etc/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/enabled
EOF
$ echo never > /sys/kernel/mm/transparent_hugepage/enabled

# kubelet中允许修改pod的net.core.somaxconn内核参数
$ cat /etc/systemd/system/kubelet.service
...
ExecStart=/usr/local/bin/kubelet \
   ...
   --allowed-unsafe-sysctls=net.core.somaxconn \
   ...
   
# 修改pod的net.core.somaxconn内核参数
$ kubectl -n demo get statefulsets  redis-redis-cluster -o yaml
...
podSpec:
	securityContext:
    sysctls:
    - name: net.core.somaxconn
    value: "20480"
...

# 启用kubelet的静态绑核开关
$ cat /etc/systemd/system/kubelet.service
...
ExecStart=/usr/local/bin/kubelet \
  ...
  --feature-gates=CPUManager=true \
  --cpu-manager-policy=static \
  --system-reserved=cpu=2,memory=500Mi,ephemeral-storage=1Gi \
  ...

# pod的resources.limits.cpu及resources.requests.cpu设置为相同的整数
$ kubectl -n demo get statefulsets  redis-redis-cluster -o yaml
...
podSpec:
	resources:
    limits:
      cpu: "1"
      ...
    requests:
      cpu: "1"
      ...
...

# 绑定网卡软中断至CPU0-CPU7
$ cat scripts/bind_nic_softirq.sh
#!/bin/bash
set -e -u
systemctl stop irqbalance.service
nic_name=enp5s0f0
irq_nos=$(grep "${nic_name}-TxRx" /proc/interrupts | awk '{print $1, $NF}' | awk -F ':' '{print $1}')
dec_value=1
for irq_no in ${irq_nos[*]}; do
  cpu_smp_affinity=$(printf '%x' ${dec_value})
  echo ${cpu_smp_affinity}  > /proc/irq/${irq_no}/smp_affinity
  dec_value=$((2*${dec_value}))
done

bash scripts/bind_nic_softirq.sh

# 所有node节点安装ipset
$ yum install -y ipset
# 配置启动时加载ipvs相关内核模块
$ cat /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
ipvs_modules=(ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack_ipv4)
for kernel_module in ${ipvs_modules[*]}; do
    /sbin/modinfo -F filename ${kernel_module} > /dev/null 2>&1
    if [ $? -eq 0 ]; then
        /sbin/modprobe ${kernel_module}
    fi
done
$ chmod +x /etc/sysconfig/modules/ipvs.modules

# kube-proxy启用ipvs模式
$ cat /etc/systemd/system/kube-proxy.service
...
ExecStart=/usr/local/bin/kube-proxy \
  ...
  --proxy-mode=ipvs \
  --ipvs-min-sync-period=5s \
  --ipvs-sync-period=5s \
  --ipvs-scheduler=rr \
  --masquerade-all \
  ...
...

   $ ip link add link ens160 mac0 type macvlan mode bridge
   # 下面的命令一定要放在一起执行，否则中间会失去连接
   $ ip addr del 192.168.179.9/16 dev ens160 && \
     ip addr add 192.168.179.9/16 dev mac0 && \
     ip link set dev mac0 up && \
     ip route flush dev ens160 && \
     ip route flush dev mac0 && \
     ip route add 192.168.0.0/16 dev mac0 metric 0 && \
     ip route add default via 192.168.1.1 dev mac0

   # 给某些node节点打label，标记为该node节点上运行管理类pod，该node节点上使用flannel的CNI网络方案，其它node节点上使用macvlan的CNI网络方案
    $ kubectl label node 10.10.20.151 managed_node=true

    # 业务类的pod使用nodeAffinity，使之被调度到没有打了label的node节点
    $ kubectl -n demo get statefulsets redis-redis-cluster -o yaml
    ...
    podSpec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: managed_node
                operator: NotIn
                values:
                - "true"