7.Prometheus监控进阶之自定义监控业务应用

pip install prometheus_client
pip install flask

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Author:WeiyiGeek
# Desc: Hello World 实例采用Python暴露Prometheus
import http.server
from prometheus_client import start_http_server
from time import sleep
class Hello(http.server.BaseHTTPRequestHandler): 
  def do_GET(self):
    self.send_response(200,'ok')
    self.end_headers()
    self.wfile.write(b"Hello World! - Python Promethus_Client!")
  
if __name__ == '__main__':
  print("Hello World, Start Prometheus Client Server!")
  start_http_server(8000)
  server = http.server.HTTPServer(('0.0.0.0',8001),Hello)
  server.serve_forever()

# -*- coding:utf-8 -*-
# Author:WeiyiGeek
# Desc: Hello World 实例采用 prometheus_client WSGI 暴露Prometheus

from prometheus_client import make_wsgi_app
from wsgiref.simple_server import make_server

metrics_app = make_wsgi_app()

def HelloWSGI(environ, start_fn):
  # 注意此/metrics可以自定义控制
  if environ['PATH_INFO'] == '/metrics':
    return metrics_app(environ, start_fn)

  start_fn('200 OK', [])
  return [b'Hello World! - Python Promethus_Client WSGI!']

if __name__ == '__main__':
  print("Hello World!, Start Prometheus Client Server!")
  httpd = make_server('0.0.0.0', 8000, HelloWSGI)
  httpd.serve_forever()

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Author: WeiyiGeek
# Desc: 演示在测控中使用 Counter 数据类型记录指标
import http.server
import random
from prometheus_client import start_http_server,Counter

# REQUESTS 记录请求 / 返回的次数(注意第二个参数是注释)
REQUESTS = Counter('http_server_total', 'http server requested frequency count.')
# EXCEPTIONS 使用上下文管理器对异常情况进行次数
EXCEPTION = Counter('http_server_execption_total', 'http server requested execption frequency count.')
EXCEPTIONS = Counter('http_server_execptions_total', 'http server requested execptions frequency count.')
# MONEYS 统计以欧元结算的销售额
MONEYS = Counter('http_server_euro_moneys_total','Euros made serving Http Server.')

class Hello(http.server.BaseHTTPRequestHandler): 
  # 作为函数的装饰器
  @EXCEPTIONS.count_exceptions()
  def do_GET(self):
    # 记录每请求一次便会+1
    REQUESTS.inc()
    # 模块中自带异常计数功能, 它不会干扰应用程序的逻辑(建议采用修饰器的方式)
    # with EXCEPTIONS.count_exceptions():
    #   if random.randint(0,10) < 1:
    #     raise Exception
    # 以欧元结算的销售额
    euros = random.randint(999,9999)
    MONEYS.inc(euros)
    self.send_response(200)
    self.end_headers()
    self.wfile.write("Hello World! Prometheus Counter Example for {} euros.".format(euros).encode())
  
if __name__ == '__main__':
  print("Hello World, Start Prometheus Client /metrics Server Port 8000!")
  start_http_server(8000)
  server = http.server.HTTPServer(('0.0.0.0',8001),Hello)
  server.serve_forever()

# HELP http_server_total http server requested frequency count.
# TYPE http_server_total counter
http_server_total 23.0
# HELP http_server_created http server requested frequency count.
# TYPE http_server_created gauge
http_server_created 1.6232518107397504e+09
# HELP http_server_execption_total http server requested execption frequency count.
# TYPE http_server_execption_total counter
http_server_execption_total 0.0
# HELP http_server_execption_created http server requested execption frequency count.
# TYPE http_server_execption_created gauge
http_server_execption_created 1.6232518107397504e+09
# HELP http_server_execptions_total http server requested execptions frequency count.
# TYPE http_server_execptions_total counter
http_server_execptions_total 4.0
# HELP http_server_execptions_created http server requested execptions frequency count.
# TYPE http_server_execptions_created gauge
http_server_execptions_created 1.6232518107397504e+09
# HELP http_server_euro_moneys_total Euros made serving Http Server.
# TYPE http_server_euro_moneys_total counter
http_server_euro_moneys_total 90706.0
# HELP http_server_euro_moneys_created Euros made serving Http Server.
# TYPE http_server_euro_moneys_created gauge
http_server_euro_moneys_created 1.6232518107397504e+09

scrape_configs:
- job_name: control
  honor_timestamps: true
  scrape_interval: 15s
  scrape_timeout: 10s
  metrics_path: /metrics
  scheme: http
  follow_redirects: true
  static_configs:
  - targets:
    - 100.201.12.103:8000

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Author: WeiyiGeek
# Desc: 演示在测控中使用 Counter 数据类型记录指标
import http.server
import time
from prometheus_client import Gauge, start_http_server

# INPROGRESS / LAST 跟踪正在处理中的请求调用数目以及最后一个调用完成时间.
INPROGRESS = Gauge('app_inprogress', 'number of http server in progress')
LAST = Gauge('app_last_seconds', 'The last time a app was Served')

# INPROGRESS_Modify 此种方式也可跟踪正在处理中的请求其调用完成时间.
INPROGRESSMODIFY = Gauge('app_inprogress_modify', 'number of http server in progress modify')
LASTTIME = Gauge('app_last_time_seconds', 'The last time a app was Served')

# TIME 回调函数使用, 例如用来返回当前时间带有`set_functiuon()`简单示例
TIME = Gauge('current_time_seconds','The Current Time')

class Hello(http.server.BaseHTTPRequestHandler): 
  @INPROGRESSMODIFY.track_inprogress()
  def do_GET(self):
    INPROGRESS.inc(3)  # +3
    self.send_response(200)
    self.end_headers()
    self.wfile.write(b"Hello World! Prometheus Gauge Type Example")
    LAST.set(time.time())
    LASTTIME.set_to_current_time()
    # 每次都会进行变化
    TIME.set_function(lambda: time.time())
    INPROGRESS.dec()  # -1
    print("End Time: {}".format(time.asctime(time.localtime(time.time()))))
  
if __name__ == '__main__':
  print("Hello World, Start Prometheus Client /metrics Server!\nServer: 127.0.0.1:8000/metrics ")
  start_http_server(8000)
  server = http.server.HTTPServer(('0.0.0.0',8001),Hello)
  server.serve_forever()

# HELP app_inprogress number of http server in progress
# TYPE app_inprogress gauge
app_inprogress 2.0    # 每次访问 (+3-1)
# HELP app_last_seconds The last time a app was Served
# TYPE app_last_seconds gauge
app_last_seconds 1.6233011413151145e+09
# HELP app_inprogress_modify number of http server in progress modify
# TYPE app_inprogress_modify gauge
app_inprogress_modify 0.0
# HELP app_last_time_seconds The last time a app was Served
# TYPE app_last_time_seconds gauge
app_last_time_seconds 1.6233011413151145e+09
# HELP current_time_seconds The Current Time
# TYPE current_time_seconds gauge
current_time_seconds 1.6233011425841126e+09

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Author: WeiyiGeek
# Desc: 演示在测控中使用 Summary 数据类型记录指标
import http.server
import time
from prometheus_client import Summary,start_http_server

# LATENCY 跟踪app程序的耗时。
LATENCY = Summary('app_latency_seconds','app 程序耗时统计单位秒')
# LATENCYMODIFY 采用修饰器来跟踪app程序的耗时。
LATENCYMODIFY = Summary('app_latencymodify_seconds','采用修饰器来验证 app 程序耗时统计单位秒')

class Hello(http.server.BaseHTTPRequestHandler):
  @LATENCYMODIFY.time()
  def do_GET(self):
    # 程序起始时间
    start = time.time()
    self.send_response(200)
    self.end_headers()
    self.wfile.write(b"Hello World! Prometheus Summary Example!")
    LATENCY.observe(time.time() - start)  

if __name__ == '__main__':
  print("Hello World, Start Prometheus Client /metrics Server!\nServer: 127.0.0.1:8000/metrics")
  start_http_server(8000)
  server = http.server.HTTPServer(('0.0.0.0',8001),Hello)
  server.serve_forever()

# HELP app_latency_seconds app 程序耗时统计单位秒
# TYPE app_latency_seconds summary
app_latency_seconds_count 15.0                # app 页面请求的次数
app_latency_seconds_sum 0.005997180938720703  # 每次 app 页面生成到展示的时间之和
# HELP app_latency_seconds_created app 程序耗时统计单位秒
# TYPE app_latency_seconds_created gauge
app_latency_seconds_created 1.6233078021234589e+09
# HELP app_latencymodify_seconds 采用修饰器来验证 app 程序耗时统计单位秒
# TYPE app_latencymodify_seconds summary
app_latencymodify_seconds_count 15.0
app_latencymodify_seconds_sum 0.006268700000005012
# HELP app_latencymodify_seconds_created 采用修饰器来验证 app 程序耗时统计单位秒
# TYPE app_latencymodify_seconds_created gauge
app_latencymodify_seconds_created 1.6233078021234589e+09

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Author: WeiyiGeek
# Desc: 演示在测控中使用 Histogram 数据类型记录指标
import http.server
import time
from prometheus_client import Histogram,start_http_server

# LATENCYTIME 使用时间函数来跟踪延时(Histogram)
LATENCYTIME = Histogram('app_latency_time_seconds','app 程序访问延迟时间统计',buckets=[0.0001 * (2**x) for x in range(1,10)])

class Hello(http.server.BaseHTTPRequestHandler):
  @LATENCYTIME.time()
  def do_GET(self):
    self.send_response(200)
    self.end_headers()
    self.wfile.write(b"Hello World! Prometheus Histogram Example!")

if __name__ == '__main__':
  print("Hello World, Start Prometheus Client /metrics Server!\nServer: 127.0.0.1:8000/metrics")
  start_http_server(8000)
  server = http.server.HTTPServer(('0.0.0.0',8001),Hello)
  server.serve_forever()

# HELP app_latency_time_seconds app 程序访问延迟时间统计
# TYPE app_latency_time_seconds histogram
app_latency_time_seconds_bucket{le="0.0002"} 0.0
app_latency_time_seconds_bucket{le="0.0004"} 31.0
app_latency_time_seconds_bucket{le="0.0008"} 51.0
app_latency_time_seconds_bucket{le="0.0016"} 56.0
app_latency_time_seconds_bucket{le="0.0032"} 57.0
app_latency_time_seconds_bucket{le="0.0064"} 57.0
app_latency_time_seconds_bucket{le="0.0128"} 57.0
app_latency_time_seconds_bucket{le="0.0256"} 57.0
app_latency_time_seconds_bucket{le="0.0512"} 57.0
app_latency_time_seconds_bucket{le="+Inf"} 57.0
app_latency_time_seconds_count 57.0
app_latency_time_seconds_sum 0.027456399999900682
# HELP app_latency_time_seconds_created app 程序访问延迟时间统计
# TYPE app_latency_time_seconds_created gauge
app_latency_time_seconds_created 1.6233113814601374e+09

# app_latency_time_seconds_bucket{le="0.0016"} 56.0
# app_latency_time_seconds_bucket{le="+Inf"} 57.0
app_latency_time_seconds_bucket{le="0.0016"} / ignoring(le) app_latency_time_seconds_bucket{le="+Inf"}   
{instance="10.20.172.103:8000", job="control"}	0.9824561403508771  # => 56 / 57

from prometheus_client import Counter
c = Counter('my_failures', 'Description of counter')
c.inc()     # Increment by 1
c.inc(1.6)  # Increment by given value

# Example
from prometheus_client import Gauge
g = Gauge('my_inprogress_requests', 'Description of gauge')
g.inc()      # Increment by 1
g.dec(10)    # Decrement by given value
g.set(4.2)   # Set to a given value

# 设置 Labels
from prometheus_client import Counter
c = Counter('my_requests_total', 'HTTP Failures', ['method', 'endpoint'])
c.labels('get', '/').inc()
c.labels('post', '/submit').inc()
# Labels can also be passed as keyword-arguments:
from prometheus_client import Counter
c = Counter('my_requests_total', 'HTTP Failures', ['method', 'endpoint'])
c.labels(method='get', endpoint='/').inc()
c.labels(method='post', endpoint='/submit').inc()

# Gauge example:
from prometheus_client import Gauge
g = Gauge('my_inprogress_requests', 'Description of gauge',['mylabelname'])
g.labels(mylabelname='labelname').set(30)

import psutil
import socket
from prometheus_client import Gauge,start_http_server
from time import sleep

g = Gauge('cup_use_percent_test_metric', 'Description of gauge',['hostip'])
host_ip = socket.gethostbyname(socket.getfqdn(socket.gethostname()))      # 获取本机IP

def get_cup_use():
    cup_use_percent = psutil.cpu_percent(0.5)      # 获取CPU使用率
    g.labels(hostip=host_ip).set(cup_use_percent)  # 本机IP传入labels，CPU使用率传入value

if __name__ == '__main__':
  start_http_server(8006)                          # 启动8006端口
  while True:
    get_cup_use()
    sleep(10)

pip install prometheus-client

# (1) Python 封装

#!/usr/bin/env python
# -*- coding:utf-8 -*-
from prometheus_client import Counter, Gauge, Summary
from prometheus_client.core import CollectorRegistry
from prometheus_client.exposition import choose_encoder

class Monitor:
    def __init__(self):
    # 注册收集器&最大耗时map
    self.collector_registry = CollectorRegistry(auto_describe=False)
    self.request_time_max_map = {}

    # 接口调用summary统计
    self.http_request_summary = Summary(name="http_server_requests_seconds",
                                   documentation="Num of request time summary",
                                   labelnames=("method", "code", "uri"),
                                   registry=self.collector_registry)
    # 接口最大耗时统计
    self.http_request_max_cost = Gauge(name="http_server_requests_seconds_max",
                                  documentation="Number of request max cost",
                                  labelnames=("method", "code", "uri"),
                                  registry=self.collector_registry)

    # 请求失败次数统计
    self.http_request_fail_count = Counter(name="http_server_requests_error",
                                      documentation="Times of request fail in total",
                                      labelnames=("method", "code", "uri"),
                                      registry=self.collector_registry)

    # 模型预测耗时统计
    self.http_request_predict_cost = Counter(name="http_server_requests_seconds_predict",
                                        documentation="Seconds of prediction cost in total",
                                        labelnames=("method", "code", "uri"),
                                        registry=self.collector_registry)
    # 图片下载耗时统计
    self.http_request_download_cost = Counter(name="http_server_requests_seconds_download",
                                         documentation="Seconds of download cost in total",
                                         labelnames=("method", "code", "uri"),
                                         registry=self.collector_registry)

    # 获取/metrics结果
    def get_prometheus_metrics_info(self, handler):
        encoder, content_type = choose_encoder(handler.request.headers.get('accept'))
        handler.set_header("Content-Type", content_type)
        handler.write(encoder(self.collector_registry))
        self.reset_request_time_max_map()

    # summary统计
    def set_prometheus_request_summary(self, handler):
        self.http_request_summary.labels(handler.request.method, handler.get_status(), handler.request.path).observe(handler.request.request_time())
        self.set_prometheus_request_max_cost(handler)

    # 自定义summary统计
    def set_prometheus_request_summary_customize(self, method, status, path, cost_time):
        self.http_request_summary.labels(method, status, path).observe(cost_time)
        self.set_prometheus_request_max_cost_customize(method, status, path, cost_time)

    # 失败统计
    def set_prometheus_request_fail_count(self, handler, amount=1.0):
        self.http_request_fail_count.labels(handler.request.method, handler.get_status(), handler.request.path).inc(amount)

    # 自定义失败统计
    def set_prometheus_request_fail_count_customize(self, method, status, path, amount=1.0):
        self.http_request_fail_count.labels(method, status, path).inc(amount)

    # 最大耗时统计
    def set_prometheus_request_max_cost(self, handler):
        requset_cost = handler.request.request_time()
        if self.check_request_time_max_map(handler.request.path, requset_cost):
            self.http_request_max_cost.labels(handler.request.method, handler.get_status(), handler.request.path).set(requset_cost)
            self.request_time_max_map[handler.request.path] = requset_cost

    # 自定义最大耗时统计
    def set_prometheus_request_max_cost_customize(self, method, status, path, cost_time):
        if self.check_request_time_max_map(path, cost_time):
            self.http_request_max_cost.labels(method, status, path).set(cost_time)
            self.request_time_max_map[path] = cost_time

    # 预测耗时统计
    def set_prometheus_request_predict_cost(self, handler, amount=1.0):
        self.http_request_predict_cost.labels(handler.request.method, handler.get_status(), handler.request.path).inc(amount)

    # 自定义预测耗时统计
    def set_prometheus_request_predict_cost_customize(self, method, status, path, cost_time):
        self.http_request_predict_cost.labels(method, status, path).inc(cost_time)

    # 下载耗时统计
    def set_prometheus_request_download_cost(self, handler, amount=1.0):
        self.http_request_download_cost.labels(handler.request.method, handler.get_status(), handler.request.path).inc(amount)

    # 自定义下载耗时统计
    def set_prometheus_request_download_cost_customize(self, method, status, path, cost_time):
        self.http_request_download_cost.labels(method, status, path).inc(cost_time)

    # 校验是否赋值最大耗时map
    def check_request_time_max_map(self, uri, cost):
        if uri not in self.request_time_max_map:
            return True
        if self.request_time_max_map[uri] < cost:
            return True
        return False

    # 重置最大耗时map
    def reset_request_time_max_map(self):
        for key in self.request_time_max_map:
            self.request_time_max_map[key] = 0.0


# (2) 封装调用
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import tornado
import tornado.ioloop
import tornado.web
import tornado.gen
from datetime import datetime
from tools.monitor import Monitor

global g_monitor

class ClassifierHandler(tornado.web.RequestHandler):
    def post(self):
        # TODO Something you need
        # work....
        # 统计Summary，包括请求次数和每次耗时
        g_monitor.set_prometheus_request_summary(self)
        self.write("OK")


class PingHandler(tornado.web.RequestHandler):
    def head(self):
        print('INFO', datetime.now(), "/ping Head.")
        g_monitor.set_prometheus_request_summary(self)
        self.write("OK")

    def get(self):
        print('INFO', datetime.now(), "/ping Get.")
        g_monitor.set_prometheus_request_summary(self)
        self.write("OK")


class MetricsHandler(tornado.web.RequestHandler):
    def get(self):
        print('INFO', datetime.now(), "/metrics Get.")
    g_monitor.set_prometheus_request_summary(self)
    # 通过Metrics接口返回统计结果
    	g_monitor.get_prometheus_metrics_info(self)
    

def make_app():
    return tornado.web.Application([
        (r"/ping?", PingHandler),
        (r"/metrics?", MetricsHandler),
        (r"/work?", ClassifierHandler)
    ])

if __name__ == "__main__":
    g_monitor = Monitor()
  
  app = make_app()
    app.listen(port)
    tornado.ioloop.IOLoop.current().start()

sudo tee /usr/lib/systemd/system/pushgateway.service <<'EOF'
[Unit]
Description=Prometheus pushgateway Server Systemd
Documentation=https://prometheus.io
After=network.target

[Service]
Type=simple
StandardError=journal
ExecStart=/usr/local/pushgateway/pushgateway --web.listen-address=:9091
Restart=on-failure
RestartSec=3s

[Install]
WantedBy=multi-user.target
EOF
sudo systemctl daemon-reload && systemctl restart pushgateway.service

- job_name: 'pushgateway'
    honor_labels: true
    static_configs:
    - targets: ['localhost:9091']

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Author: WeiyiGeek
# Desc: 演示在测控中使用Python自定义指标并将采集的数据推送到Pushgateway上

import requests,random,pprint
from prometheus_client import CollectorRegistry, Gauge, pushadd_to_gateway, registry

pushgateway_ip = "10.10.107.249"
pushgateway_port = "9091"

# 方式1.创建一个测控批处理任务并且将指标数据推送给Pushgateway。
registry = CollectorRegistry()
durationtime = Gauge('app_job_duration_time_seconds','duration of my batch in seconds', registry=registry)
try:
  with durationtime.time():
    # Guage(metric_name,HELP,labels_name,registry=registry)
    suc = Gauge('app_job_sucess_seconds','Last time my job successfully finished',['dst_ip','name'],registry=registry)
    suc.labels('192.168.1.1','app').set(random.randrange(999,9999))
    suc.labels('192.168.1.11','weblogic').dec(2)  #dec递减2
    suc.labels('192.168.1.12','nginx').inc()  #inc递增，默认增1
finally:
  pushadd_to_gateway(pushgateway_ip+":"+pushgateway_port,job='app',registry=registry)

# 方式2.采用request模块进行指标数据上传。
quota = """
# TYPE test_metric counter
test_metric{label="test"} %s
# TYPE another_metric gauge
# HELP another_metric Just an example.
another_metric{label="prod"} %s
"""%(random.randrange(1,100),random.random())
# quota_bin = binascii.hexlify(quota)
res = requests.post(url="http://"+pushgateway_ip+":"+pushgateway_port+"/metrics/job/app/custom_instance/test_instance",data=quota,headers={'Content-Type': 'application/octet-stream'})
pprint.pprint(res)
pprint.pprint(res.status_code)

# -- 执行结果 ---
<Response [200]>
200

# - 1.推送单个或者多个监控指标到网关
echo "name_count 1" | curl --data-binary @- http://10.10.107.249:9091/metrics/job/curl_job
cat <<EOF | curl --data-binary @- http://10.10.107.249:9091/metrics/job/curl_job/app/weiyigeek
# TYPE app_init_total counter
# HELP This is app initialization count 
app_init_total 1314
# TYPE app_init_metric gauge
# HELP app_init_metric Just an example.
app_init_metric 1024.16
EOF

# -2.把多个指标写入到文件里通过curl的@读取文件来上传指标
tee metrics <<'EOF'
http_request_total{code="200",domain="weiyigeek.top"} 276683
http_request_total{code="400",domain="weiyigeek.top"} 0
http_request_total{code="408",domain="weiyigeek.top"} 7
http_request_total{code="401",domain="weiyigeek.top"} 0
http_request_total{schema="http",domain="weiyigeek.top"} 277725
http_request_total{schema="https",domain="weiyigeek.top"} 0
http_request_time{code="total",domain="weiyigeek.top"} 76335.809000
http_request_uniqip{domain="weiyigeek.top"} 216944
http_request_maxip{clientip="172.27.0.12",domain="weiyigeek.top"} 81
EOF
curl -XPOST --data-binary @metrics http://10.10.107.249:9091/metrics/job/curl_job/app/my_blog/site/weiyigeek.top

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Author: WeiyiGeek
# Desc: 演示在测控中为指标设置多个标签
import http.server
import random
from prometheus_client import Gauge, Summary, start_http_server,Counter

# - REQUESTS 记录网站各访问路径与请求方法合计次数(多标签)
REQUESTS = Counter('http_requested_total', '网站各访问路径与请求方法合计',['path', 'method'])
# - ENUM 记录枚举值出现次数(多标签)
ENUM = Counter('http_enum_total', '请求网站后枚举值的此时',['index','name'])
DEMO_ARAAY = ["one","two","there","four"]
# - MODIFIER 使用带有标签的修饰器(多标签)
MODIFIER = Summary('http_modifier_seconds', '请求网站中使用带有标签的修饰器', ['index','name'])
modifier = MODIFIER.labels('MODIFIER','weiyigeek')
# - DEMOINFO - 暴露指标的指定信息
python_info = { "implementation": "CPython", "major": "3", "minor": "7", "patchlevel": "0", "version": "3.8.0" }
DEMOINFO = Gauge('demo_info','暴露指标的指定信息',labelnames=python_info.keys())
DEMOINFO.labels(**python_info).set(1)


class Hello(http.server.BaseHTTPRequestHandler): 
  @modifier.time()
  def do_GET(self):
    REQUESTS.labels(self.path, self.command).inc()
    ENUM.labels(DEMO_ARAAY[random.randint(0,DEMO_ARAAY.__len__() - 1)], 'ENUM').inc()
    self.send_response(200)
    self.end_headers()
    self.wfile.write(b"Hello World! Multi label Example!")
  
if __name__ == '__main__':
  print("Start Prometheus Client /metrics Server!")
  start_http_server(8000)
  server = http.server.HTTPServer(('0.0.0.0',8001),Hello)
  server.serve_forever()

# HELP http_requested_total 网站各访问路径与请求方法合计
# TYPE http_requested_total counter
http_requested_total{method="GET",path="/number"} 4.0
http_requested_total{method="GET",path="/favicon.ico"} 7.0
http_requested_total{method="GET",path="/result"} 1.0
http_requested_total{method="GET",path="/"} 2.0
http_requested_total{method="GET",path="/api/"} 5.0
# HELP http_requested_created 网站各访问路径与请求方法合计
# TYPE http_requested_created gauge
http_requested_created{method="GET",path="/number"} 1.623854061065871e+09
http_requested_created{method="GET",path="/favicon.ico"} 1.6238540611298337e+09
http_requested_created{method="GET",path="/result"} 1.6238540711289563e+09
http_requested_created{method="GET",path="/"} 1.6238540782242467e+09
http_requested_created{method="GET",path="/api/"} 1.6238541072589352e+09
# HELP http_enum_total 请求网站后枚举值的此时
# TYPE http_enum_total counter
http_enum_total{index="two",name="ENUM"} 4.0
http_enum_total{index="there",name="ENUM"} 4.0
http_enum_total{index="one",name="ENUM"} 6.0
http_enum_total{index="four",name="ENUM"} 5.0
# HELP http_enum_created 请求网站后枚举值的此时
# TYPE http_enum_created gauge
http_enum_created{index="two",name="ENUM"} 1.623854061065871e+09
http_enum_created{index="there",name="ENUM"} 1.6238540611298337e+09
http_enum_created{index="one",name="ENUM"} 1.623854061224779e+09
http_enum_created{index="four",name="ENUM"} 1.6238540614066763e+09
# HELP http_modifier_seconds 请求网站中使用带有标签的修饰器
# TYPE http_modifier_seconds summary
http_modifier_seconds_count{index="MODIFIER",name="weiyigeek"} 19.0
http_modifier_seconds_sum{index="MODIFIER",name="weiyigeek"} 0.008954100000021725
# HELP http_modifier_seconds_created 请求网站中使用带有标签的修饰器
# TYPE http_modifier_seconds_created gauge
http_modifier_seconds_created{index="MODIFIER",name="weiyigeek"} 1.6238540450573144e+09
# HELP demo_info 暴露指标的指定信息
# TYPE demo_info gauge
demo_info{implementation="CPython",major="3",minor="7",patchlevel="0",version="3.8.0"} 1.0

# - 1.只会删除标签只有job="curl_job"的指标
curl -X DELETE http://10.10.107.249:9091/metrics/job/curl_job/

# - 2.删除指定工作任务名以及指标标签的指标
curl -X DELETE http://10.10.107.249:9091/metrics/job/curl_job/app/weiyigeek

# - 3.当pushgateway启动时设置--web.enable-admin-api参数就可以一键清空所有指标了
curl -X PUT http://10.10.107.249:9091/api/v1/admin/wipe

# - 通用占位符定义如下：
  * <boolean>: a boolean that can take the values true or false
  * <int>: a regular integer
  * <duration>: a duration matching the regular expression [0-9]+(ms|[smhdwy])
  * <filename>: a valid path in the current working directory
  * <string>: a regular string
  * <secret>: a regular string that is a secret, such as a password
  * <regex>: a regular expression

# - exporter configuration file format
Module
  # The protocol over which the probe will take place (http, tcp, dns, icmp).
  prober: <prober_string>
  # How long the probe will wait before giving up.
  [ timeout: <duration> ]
  # The specific probe configuration - at most one of these should be specified.
  [ http: <http_probe> ]
  [ tcp: <tcp_probe> ]
  [ dns: <dns_probe> ]
  [ icmp: <icmp_probe> ]

mkdir /usr/local/blackbox
tee /usr/local/blackbox/blackbox.yml <<'EOF'
modules:
  icmp_example:
    prober: icmp
    timeout: 5s
    icmp:
      preferred_ip_protocol: "ip4"
#     source_ip_address: "127.0.0.1"  # 多张网卡时指定
  tcp_example:
    prober: tcp
    timeout: 5s
  tcp_ssh_example: 
    prober: tcp
    timeout: 5s
    tcp:
      query_response:
      - expect: "^SSH-2.0-"               # 验证监控端是否以SSH进行响应 
  tcp_tls_example:
    prober: tcp
    timeout: 5s
    tcp:
      tls: true
  http_example:
    prober: http
    timeout: 5s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: []  # Defaults to 2xx
      method: GET
      headers:
        User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0"
      # 如果存在SSL，则探测失败。
      fail_if_ssl: false
      tls_config:
        insecure_skip_verify: false
      preferred_ip_protocol: "ip4" # defaults to "ip6"
  https_example:
    prober: http
    timeout: 5s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: []  # Defaults to 2xx
      method: GET
      headers:
        User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0"
      # 如果不存在SSL，则探测失败。
      fail_if_not_ssl: false 
      tls_config:
        insecure_skip_verify: false
      preferred_ip_protocol: "ip4" # defaults to "ip6"
      ip_protocol_fallback: false  # no fallback to "ip6"
  dns_example:
    prober: dns
    dns:
      query_name: "www.weiyigeek.top"
      query_type: "A"
      valid_rcodes:
      - NOERROR
  dns_tcp_example:
    prober: dns
    dns:
      transport_protocol: "tcp"    # defaults to "udp"
      preferred_ip_protocol: "ip4" # defaults to "ip6"
      query_name: "www.prometheus.io"
  dns_mx_example:
    prober: dns
    dns:
      query_name: "weiyigeek.top"
      query_type: "MX"
      validate_answer_rrs:
        fail_if_not_matches_regexp:
        - ".+"
EOF

docker run --rm -d -p 9115:9115 --name blackbox_exporter -v /usr/local/blackbox:/config prom/blackbox-exporter:master --config.file=/config/blackbox.yml
d571d7587a031d01fabd82c780d8bb90bba4c88a06834e5e8b6f081a99442db9

# - 结果说明
# HELP probe_dns_lookup_time_seconds Returns the time taken for probe dns lookup in seconds
# TYPE probe_dns_lookup_time_seconds gauge
probe_dns_lookup_time_seconds 2.5798e-05
# HELP probe_duration_seconds Returns how long the probe took to complete in seconds
# TYPE probe_duration_seconds gauge
probe_duration_seconds 0.006506971  # 整个探测花费时间包括DNS解析时间
# HELP probe_icmp_duration_seconds Duration of icmp request by phase
# TYPE probe_icmp_duration_seconds gauge
probe_icmp_duration_seconds{phase="resolve"} 2.5798e-05  
probe_icmp_duration_seconds{phase="rtt"} 0.00625726
probe_icmp_duration_seconds{phase="setup"} 0.000126281
# HELP probe_icmp_reply_hop_limit Replied packet hop limit (TTL for ipv4)
# TYPE probe_icmp_reply_hop_limit gauge
probe_icmp_reply_hop_limit 113
# HELP probe_ip_addr_hash Specifies the hash of IP address. It's useful to detect if the IP address changes.
# TYPE probe_ip_addr_hash gauge
probe_ip_addr_hash 2.372708602e+09  # 监测目标地址Hash
# HELP probe_ip_protocol Specifies whether probe ip protocol is IP4 or IP6
# TYPE probe_ip_protocol gauge
probe_ip_protocol 4  # 使用IP协议版本
# HELP probe_success Displays whether or not the probe was a success
# TYPE probe_success gauge
probe_success 1      # 成功返回1，否则返回0

# TYPE probe_failed_due_to_regex gauge
probe_failed_due_to_regex 0
# TYPE probe_success gauge
probe_success 1   # 成功

# HELP probe_ip_protocol Specifies whether probe ip protocol is IP4 or IP6
# TYPE probe_ip_protocol gauge
probe_ip_protocol 4
# HELP probe_success Displays whether or not the probe was a success
# TYPE probe_success gauge
probe_success 1   # 匹配成功

# HELP probe_ssl_earliest_cert_expiry Returns earliest SSL cert expiry date
# TYPE probe_ssl_earliest_cert_expiry gauge
probe_ssl_earliest_cert_expiry 1.627277462e+09   # TLS/SSL证书到期时间
# HELP probe_ssl_last_chain_expiry_timestamp_seconds Returns last SSL chain expiry in unixtime
# TYPE probe_ssl_last_chain_expiry_timestamp_seconds gauge
probe_ssl_last_chain_expiry_timestamp_seconds 1.627277462e+09    # 返回unixtime中最后一个SSL链到期时间
# HELP probe_ssl_last_chain_info Contains SSL leaf certificate information
# TYPE probe_ssl_last_chain_info gauge
probe_ssl_last_chain_info{fingerprint_sha256="2ed189349f818f3414132ebea309e36f620d78a0507a2fa523305f275062d73c"} 1  # 证书链信息
# HELP probe_success Displays whether or not the probe was a success
# TYPE probe_success gauge
probe_success 1   # 成功标志
# HELP probe_tls_version_info Returns the TLS version used, or NaN when unknown
# TYPE probe_tls_version_info gauge
probe_tls_version_info{version="TLS 1.2"} 1  # 证书版本

# HELP probe_dns_lookup_time_seconds Returns the time taken for probe dns lookup in seconds
# TYPE probe_dns_lookup_time_seconds gauge
probe_dns_lookup_time_seconds 0.186725698
# HELP probe_failed_due_to_regex Indicates if probe failed due to regex
# TYPE probe_failed_due_to_regex gauge
probe_failed_due_to_regex 0
# HELP probe_http_content_length Length of http content response
# TYPE probe_http_content_length gauge
probe_http_content_length -1
# HELP probe_duration_seconds Returns how long the probe took to complete in seconds
# TYPE probe_duration_seconds gauge
probe_duration_seconds 2.142812294
# HELP probe_http_duration_seconds Duration of http request by phase, summed over all redirects
# TYPE probe_http_duration_seconds gauge
probe_http_duration_seconds{phase="connect"} 1.209361663
probe_http_duration_seconds{phase="processing"} 0.250921647
probe_http_duration_seconds{phase="resolve"} 0.186725698
probe_http_duration_seconds{phase="tls"} 0
probe_http_duration_seconds{phase="transfer"} 0.495566665
# HELP probe_http_redirects The number of redirects
# TYPE probe_http_redirects gauge
probe_http_redirects 0
# HELP probe_http_ssl Indicates if SSL was used for the final redirect
# TYPE probe_http_ssl gauge
probe_http_ssl 0
# HELP probe_http_status_code Response HTTP status code
# TYPE probe_http_status_code gauge
probe_http_status_code 200
# HELP probe_http_uncompressed_body_length Length of uncompressed response body
# TYPE probe_http_uncompressed_body_length gauge
probe_http_uncompressed_body_length 22279
# HELP probe_http_version Returns the version of HTTP of the probe response
# TYPE probe_http_version gauge
probe_http_version 1.1  # HTTP 协议版本
# HELP probe_ip_addr_hash Specifies the hash of IP address. It's useful to detect if the IP address changes.
# TYPE probe_ip_addr_hash gauge
probe_ip_addr_hash 2.341618423e+09
# HELP probe_ip_protocol Specifies whether probe ip protocol is IP4 or IP6
# TYPE probe_ip_protocol gauge
probe_ip_protocol 4
# HELP probe_success Displays whether or not the probe was a success
# TYPE probe_success gauge
probe_success 1          # 探测结果

# HELP probe_duration_seconds Returns how long the probe took to complete in seconds
# TYPE probe_duration_seconds gauge
probe_duration_seconds 3.301450584
# HELP probe_http_duration_seconds Duration of http request by phase, summed over all redirects
# TYPE probe_http_duration_seconds gauge
probe_http_duration_seconds{phase="connect"} 0.35759203100000003
probe_http_duration_seconds{phase="processing"} 0.39931457400000003
probe_http_duration_seconds{phase="resolve"} 2.334050778
probe_http_duration_seconds{phase="tls"} 0.196995271
probe_http_duration_seconds{phase="transfer"} 0.012739035
# HELP probe_http_redirects The number of redirects
# TYPE probe_http_redirects gauge
probe_http_redirects 1
# HELP probe_http_ssl Indicates if SSL was used for the final redirect
# TYPE probe_http_ssl gauge
probe_http_ssl 1
# HELP probe_http_status_code Response HTTP status code
# TYPE probe_http_status_code gauge
probe_http_status_code 200
# HELP probe_http_uncompressed_body_length Length of uncompressed response body
# TYPE probe_http_uncompressed_body_length gauge
probe_http_uncompressed_body_length 16377
# HELP probe_http_version Returns the version of HTTP of the probe response
# TYPE probe_http_version gauge
probe_http_version 2
# HELP probe_ssl_earliest_cert_expiry Returns earliest SSL cert expiry in unixtime
# TYPE probe_ssl_earliest_cert_expiry gauge
probe_ssl_earliest_cert_expiry 1.628424e+09
# HELP probe_ssl_last_chain_expiry_timestamp_seconds Returns last SSL chain expiry in timestamp seconds
# TYPE probe_ssl_last_chain_expiry_timestamp_seconds gauge
probe_ssl_last_chain_expiry_timestamp_seconds 1.628424e+09
# HELP probe_ssl_last_chain_info Contains SSL leaf certificate information
# TYPE probe_ssl_last_chain_info gauge
probe_ssl_last_chain_info{fingerprint_sha256="44c9e62838db98e79918c841e4b72529849e2e7e6654e337a32a74ec502edaa7"} 1
# HELP probe_success Displays whether or not the probe was a success
# TYPE probe_success gauge
probe_success 1
# HELP probe_tls_version_info Contains the TLS version used
# TYPE probe_tls_version_info gauge
probe_tls_version_info{version="TLS 1.3"} 1

# HELP probe_dns_additional_rrs Returns number of entries in the additional resource record list
# TYPE probe_dns_additional_rrs gauge
probe_dns_additional_rrs 0      # 返回附加资源记录列表中的条目数
# HELP probe_dns_answer_rrs Returns number of entries in the answer resource record list
# TYPE probe_dns_answer_rrs gauge
probe_dns_answer_rrs 2            # DNS 回答响应条目
# HELP probe_dns_authority_rrs Returns number of entries in the authority resource record list
# TYPE probe_dns_authority_rrs gauge
probe_dns_authority_rrs 0        # 返回权限资源记录列表中的条目数
# HELP probe_dns_duration_seconds Duration of DNS request by phase
# TYPE probe_dns_duration_seconds gauge
probe_dns_duration_seconds{phase="connect"} 7.9269e-05
probe_dns_duration_seconds{phase="request"} 0.072951973
probe_dns_duration_seconds{phase="resolve"} 2.9612e-05
# HELP probe_dns_lookup_time_seconds Returns the time taken for probe dns lookup in seconds
# TYPE probe_dns_lookup_time_seconds gauge
probe_dns_lookup_time_seconds 2.9612e-05  # 返回探测dns查找所用的时间（以秒为单位）
# HELP probe_duration_seconds Returns how long the probe took to complete in seconds
# TYPE probe_duration_seconds gauge
probe_duration_seconds 0.073247895   # 返回以秒为单位完成探测所用的时间
# HELP probe_success Displays whether or not the probe was a success
# TYPE probe_success gauge
probe_success 1

# HELP probe_dns_answer_rrs Returns number of entries in the answer resource record list
probe_dns_answer_rrs 1
# HELP probe_success Displays whether or not the probe was a success
probe_success 1

probe_dns_answer_rrs 2
probe_success 1

# prometheus.yml
- job_name: 'blackbox'
  metrics_path: /probe
  params:
    module: [https_example]  # Look for a HTTP 200 response.
  static_configs:
    - targets:
      - http://weiyigeek.top    # Target to probe with http.
      - https://prometheus.io   # Target to probe with https.
  relabel_configs:
    - source_labels: [__address__]
      target_label: __param_target
    - source_labels: [__param_target]
      target_label: instance
    - target_label: __address__
      replacement: 192.168.12.107:9115  # The blackbox exporter's real hostname:port.

# - prometheus.yml
  file_sd_configs:
    - files:
      - /etc/prometheus/conf.d/discovery/blackbox.yaml
      refresh_interval: 1m

# - blackbox.yaml
- targets: [ 'http://weiyigeek.top','https://blog.weiyigeek.top']
  labels: {'env': 'prod', 'appname': 'mySite', 'label': 'blog'}
- targets: [ 'https://prometheus.io']
  labels: {'env': 'test', 'appname': 'prometheus官网', 'label': 'officialwebiste'}

groups:
- name: normal
  rules:
  - alert: web_request
    expr: probe_success == 0
    for: 30s
    labels:
      severity: 'critical'
    annotations:
      summary: "站点 [ {{ $labels.instance }} ] , 应用名称: {{ $labels.appname }}, 标签来自: {{ $labels.label}}"

# Blackbox_exporter 下载
wget https://github.com/prometheus/blackbox_exporter/releases/download/v0.19.0/blackbox_exporter-0.19.0.linux-amd64.tar.gz
tar -zxvf blackbox_exporter-0.19.0.linux-amd64.tar.gz -C /usr/local/
mv /usr/local/blackbox_exporter-0.19.0.linux-amd64   /usr/local/blackbox_exporter

# 版本信息查看
cd /usr/local/blackbox_exporter
./blackbox_exporter --version
blackbox_exporter, version 0.19.0 (branch: HEAD, revision: 5d575b88eb12c65720862e8ad2c5890ba33d1ed0)
  build user:       root@2b0258d5a55a
  build date:       20210510-12:56:44
  go version:       go1.16.4
  platform:         linux/amd6

$ ls /usr/local/blackbox_exporter && cd /usr/local/blackbox_exporter
blackbox_exporter  blackbox.yml  LICENSE  NOTICE

# - 修改并添加自定义的对象后的BlackBox.yml.
$ tee /usr/local/blackbox_exporter/blackbox.yml <<'EOF'
modules:
  icmp:
    prober: icmp
    timeout: 5s
    icmp:
      preferred_ip_protocol: "ip4"
  tcp:
    prober: tcp
    timeout: 5s
  tcp_tls:
    prober: tcp
    timeout: 5s
    tcp:
      tls: true
  http_2xx:
    prober: http
    timeout: 5s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: []      # Defaults to 2xx
      method: GET
      headers:
        User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0"
      tls_config:
        insecure_skip_verify: false
      preferred_ip_protocol: "ip4" # Defaults to "ip6"
  http_post:
    prober: http
    timeout: 5s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: []       # Defaults to 2xx
      method: POST
      headers:
        User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0"
        Content-Type: application/json
      body: '{}'
      fail_if_not_ssl: false 
      tls_config:
        insecure_skip_verify: false
      preferred_ip_protocol: "ip4" # defaults to "ip6"
      ip_protocol_fallback: false  # no fallback to "ip6"
  http_basic_auth_example:
    prober: http
    timeout: 5s
    http:
      method: POST
      headers:
        Host: "login.example.com"
      basic_auth:
        username: "username"
        password: "mysecret"
EOF

sudo tee /usr/lib/systemd/system/blackbox_exporter.service <<'EOF'
[Unit]
Description=blackbox_exporter
Documentation=https://prometheus.io
After=network.target

[Service]
Type=simple
StandardError=journal
ExecStart=/usr/local/blackbox_exporter/blackbox_exporter --config.file=/usr/local/blackbox_exporter/blackbox.yml
Restart=on-failure
RestartSec=3s

[Install]
WantedBy=multi-user.target
EOF

# - 自动重启
sudo systemctl daemon-reload && systemctl restart blackbox_exporter.service
systemctl status blackbox_exporter

# - 监听进程查看
ss -lnpt | grep 9115
  # LISTEN     0      128       [::]:9115                  [::]:*                   users:(("blackbox_export",pid=1738,fd=3))

$ vi /usr/local/prometheus/prometheus.yml
# - 在配置文件中添加外网网关存活状态的job信息
  - job_name: 'ICMP-Ping'
    scrape_interval: 15s
    metrics_path: /probe
    params:
      module: [icmp]
    file_sd_configs:
    - refresh_interval: 30s
      files:
      - /etc/prometheus/backbox_icmp.yml
    relabel_configs:
    - source_labels: [__address__]
      regex: (.*)(:80)?
      target_label: __param_target
      replacement: ${1}
    - source_labels: [__param_target]
      regex: (.*)
      target_label: ping
      replacement: ${1}
    - source_labels: []
      regex: .*
      target_label: __address__
      replacement: 127.0.0.1:9115
  - job_name: 'HTTP-Check'
    scrape_interval: 30s
    metrics_path: /probe
    params:
      module: [http_2xx]
    file_sd_configs:
    - refresh_interval: 30s
      files:
      - /etc/prometheus/backbox_http.yml
    relabel_configs:
    - source_labels: [__address__]
      regex: (.*)(:80)?
      target_label: __param_target
      replacement: ${1}
    - source_labels: [__param_target]
      regex: (.*)
      target_label: http
      replacement: ${1}
    - source_labels: []
      regex: .*
      target_label: __address__
      replacement: 127.0.0.1:9115

# 以 文件的形式 进行目标发现。
# icmp 协议监控目标对象设置以及其标签设置
tee /etc/prometheus/backbox_icmp.yml <<'EOF'
- targets: [ '183.230.37.209','117.59.89.218']
  labels: {'env': 'prod','BusinessType': 'Gateway','site':'www.weiyigeek.top','msg': '官网出口IP地址'}
EOF

# http 协议监控目标对象设置以及其标签设置
tee /etc/prometheus/backbox_http.yml <<'EOF'
- targets: [ 'http://www.weiyigeek.top' ]
  labels: {'env': 'prod','BusinessType': 'Web','site': '个人官网-www.weiyigeek.top','msg': 'A10-Load Balance-VIP 地址'}
EOF

/usr/local/prometheus/promtool check config /usr/local/prometheus/prometheus.yml
Checking prometheus.yml
  SUCCESS: 2 rule files found

# - 动态重载blackbox配置
curl -XPOST http://127.0.0.1:9115/-/reload
# - 热加载Prometheus
curl -XPOST http://127.0.0.1:9090/-/reload

# Tips: 此模板需要下载饼状图插件。
wget -nv https://grafana.com/api/plugins/grafana-piechart-panel/versions/latest/download -O /tmp/grafana-piechart-panel.zip
grafana-cli plugins install grafana-piechart-panel
systemctl grafana-server restart