可观测平台-4.2: Cache/MQ/TQ 中间件告警管理

groups:
- name: redis_metrics
  rules:
  - record: redis_commands_processed_total
    expr: rate(redis_commands_processed_total[5m])

  - record: redis_memory_usage_bytes
    expr: redis_memory_used_bytes

  - record: redis_cpu_usage_percentage
    expr: rate(redis_cpu_user_seconds_total[5m]) + rate(redis_cpu_system_seconds_total[5m])

  - record: redis_net_input_bytes
    expr: rate(redis_net_input_bytes_total[5m])

  - record: redis_net_output_bytes
    expr: rate(redis_net_output_bytes_total[5m])

groups:
- name: redis_alerts
  rules:
  - alert: HighMemoryUsage
    expr: redis_memory_usage_bytes > (redis_memory_max_bytes * 0.8)
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "High Memory Usage in Redis"
      description: "Redis server {{ $labels.instance }} is using more than 80% of its configured max memory."

  - alert: HighCommandRate
    expr: redis_commands_processed_total > 10000
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High Command Rate on Redis"
      description: "Redis server {{ $labels.instance }} is processing more than 10000 commands per second."

groups:
- name: kafka_metrics
  rules:
  - record: kafka_messages_per_second
    expr: rate(kafka_server_brokertopicmetrics_messagesin_total{topic="your_topic"}[5m])

  - record: kafka_latency_seconds
    expr: kafka_server_brokertopicmetrics_totaltime_seconds{topic="your_topic"} / kafka_server_brokertopicmetrics_messagesin_total{topic="your_topic"}

  - record: kafka_queue_size
    expr: kafka_server_brokertopicmetrics_messagesin_total{topic="your_topic"} - kafka_server_brokertopicmetrics_messagesout_total{topic="your_topic"}

  - record: kafka_cpu_usage_percentage
    expr: 100 - avg by (instance)(irate(node_cpu_seconds_total{mode="idle"}[5m]))

  - record: kafka_memory_usage_bytes
    expr: node_memory_MemTotal_bytes - node_memory_MemFree_bytes

  - record: kafka_error_rate
    expr: rate(kafka_server_brokertopicmetrics_failedfetchrequests_total{topic="your_topic"}[5m]) / rate(kafka_server_brokertopicmetrics_requests_total{topic="your_topic"}[5m])

  - record: kafka_retry_count
    expr: sum(kafka_consumer_rebalance_total{topic="your_topic"})

  - record: kafka_client_connections
    expr: kafka_server_metadata_response_version{version="0.9.0.1"} / ignoring(version) count(kafka_server_metadata_response_version{version="0.9.0.1"})

  - record: kafka_client_connection_failures
    expr: sum(kafka_network_handlers_total{result="NetworkError"})

groups:
- name: kafka_alerts
  rules:
  - alert: HighLatency
    expr: kafka_latency_seconds > 0.5
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "High Latency in Kafka"
      description: "Kafka topic 'your_topic' is experiencing high latency."

  - alert: HighErrorRate
    expr: kafka_error_rate > 0.05
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High Error Rate in Kafka"
      description: "Kafka topic 'your_topic' has a high error rate."

  - alert: HighQueueSize
    expr: kafka_queue_size > 1000
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High Queue Size in Kafka"
      description: "Kafka topic 'your_topic' has a large queue size."

  - alert: ConnectionFailures
    expr: kafka_client_connection_failures > 10
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Kafka Connection Failures"
      description: "Kafka clients are experiencing connection failures."

groups:
- name: celery_metrics
  rules:
  - record: celery_task_throughput
    expr: rate(celery_task_processed_total[5m])

  - record: celery_task_latency_seconds
    expr: celery_task_processing_time_seconds

  - record: celery_queue_length
    expr: celery_queue_size

  - record: celery_memory_usage_bytes
    expr: celery_memory_used_bytes

  - record: celery_cpu_usage_percentage
    expr: 100 - avg by (instance)(irate(node_cpu_seconds_total{mode="idle"}[5m]))

  - record: celery_task_success_rate
    expr: celery_task_successful_total / celery_task_processed_total

  - record: celery_task_failure_count
    expr: celery_task_failed_total

  - record: celery_task_retry_count
    expr: celery_task_retry_total

  - record: celery_queue_service_status
    expr: celery_service_status

  - record: celery_connection_errors
    expr: celery_connection_failures_total

  - record: celery_worker_count
    expr: celery_active_workers_total

  - record: celery_worker_load
    expr: celery_worker_load

groups:
- name: celery_alerts
  rules:
  - alert: HighTaskLatency
    expr: celery_task_latency_seconds > 10
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "High Task Latency in Celery"
      description: "Celery tasks are experiencing high latency."

  - alert: HighQueueLength
    expr: celery_queue_length > 100
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High Queue Length in Celery"
      description: "Celery queue has a large number of pending tasks."

  - alert: HighMemoryUsage
    expr: celery_memory_usage_bytes > 536870912  # 512 MB
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High Memory Usage in Celery"
      description: "Celery is using more than 512 MB of memory."

  - alert: ConnectionFailures
    expr: celery_connection_errors > 10
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Celery Connection Failures"
      description: "Celery is experiencing connection failures to the backend."

  - alert: HighTaskFailureRate
    expr: celery_task_failure_count / celery_task_processed_total > 0.1
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "High Task Failure Rate in Celery"
      description: "Celery tasks are failing at a high rate."