运维降本增效！大规模k8s集群的潮汐调度

helm install kruise openkruise/kruise --version 1.6.3 --set featureGates="PodProbeMarkerGate=true"

apiVersion: apps/v1 
kind: Deployment
metadata:
  name: nginx
  namespace: hpa
  labels:
    app: nginx
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx  
  template:
    metadata:
      labels:
        app: nginx
        nginx-idle: 'xxx'   # 默认值设置为xxx，如果是闲置的时候会显示为ture
      annotations:
        controller.kubernetes.io/pod-deletion-cost: '100'  # 初始删除成本我们设置为100
    spec:
      containers:
      - name: nginx
        image: nginx:1.16.1
        ports:
        - containerPort: 80
        resources:
          requests:       # 必须设置，否则HPA无法运行
            cpu: 100m     # 设置容器运行时所需CPU资源
            memory: 200Mi # 设置容器运行时所需内存资源
          limits:         # 不是必须设置，但是设置后可以避免压测时占用过多资源
            cpu: 200m      # 限制容器CPU资源为200m,m 是毫核（milli-cores）的缩写，因此200m表示0.2核
            memory: 400Mi   # 限制容器内存资源为400Mi 
---
kind: Service
apiVersion: v1
metadata:
  name: nginx
  namespace: hpa
spec:
  type: NodePort
  selector:
    app: nginx
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: nginx-hpa
  namespace: hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: nginx
  minReplicas: 2  # Deployment可缩容的容器数量下限，需设置为大于等于1的整数
  maxReplicas: 10  # 该Deployment可扩容的容器数量上限，需大于minReplicas
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 50 # 目标资源的平均使用率，即资源使用量的平均值与其请求量之间的比例
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300
    scaleUp:
      stabilizationWindowSeconds: 300

#!/bin/bash
# 读取 /data/task.txt 文件的内容
task_status=$(cat /data/task.txt)

# 判断 task_status 的值
if [ "$task_status" -eq 0 ]; then
  # 如果值为 0，表示空闲
  exit 0
elif [ "$task_status" -eq 1 ]; then
  # 如果值为 1，表示繁忙
  exit 1
fi

apiVersion: apps.kruise.io/v1alpha1
kind: PodProbeMarker
metadata:
  name: nginx-probe
  namespace: hpa
spec:
  selector:
    matchLabels:
      app: nginx
  probes:
  - name: Idle
    containerName: nginx
    probe:
      exec:
        command:
        - /home/nginx/idle.sh
      initialDelaySeconds: 10
      timeoutSeconds: 3
      periodSeconds: 10
      successThreshold: 1
      failureThreshold: 3
    markerPolicy:
    - state: Succeeded
      labels:
        nginx-idle: 'true'
      annotations:
        controller.kubernetes.io/pod-deletion-cost: '5'
    - state: Failed
      labels:
        nginx-idle: 'false'
      annotations:
        controller.kubernetes.io/pod-deletion-cost: '10'
    podConditionType: nginx.io/idle

kubectl get pods -l app=nginx -n hpa -o json | jq '.items[] | {POD_NAME: .metadata.name, POD_DELETION_COST: .metadata.annotations["controller.kubernetes.io/pod-deletion-cost"]}'

kubectl scale deployment nginx -n hpa --replicas=1