prometheus内核

// prometheus/prometheus/cmd/prometheus/main.go

global:

  # 这两个是全局的刮取间隔以及超时设置 
  # How frequently to scrape targets by default.
  [ scrape_interval: <duration> | default = 1m ]
  # How long until a scrape request times out.
  [ scrape_timeout: <duration> | default = 10s ]

  # 多久evaluate rules一次
  [ evaluation_interval: <duration> | default = 1m ]

  # 全局的 external label, 当 prometheus (federation, remote storage, Alertmanager) 
  # 和外部交互的时候很有用 . 举个例子：当多个prometheus数据聚合到同一个federation prometheus
  # 或者 remote storage 的时候，可以加一个 id/cluster label作为区分
  external_labels:
    [ <labelname>: <labelvalue> ... ]

# 规则文件路径，规则分为两种，一种叫 recoding rule, 另一种叫 alter rule
# 他们都是 隔一段时间内部 evaluate 一次，生成新的 metrics 或者 产生 alter notification
rule_files:
  [ - <filepath_glob> ... ]

# 抓取配置，这个是配置文件的大头，各种抓取规则都在这里，下面会分解细讲
scrape_configs:
  [ - <scrape_config> ... ]

# Alertmanager相关的配置
alerting:
  alert_relabel_configs:
    [ - <relabel_config> ... ]
  alertmanagers:
    [ - <alertmanager_config> ... ]

# remote_write/read 相关的配置
remote_write:
  [ - <remote_write> ... ]
remote_read:
  [ - <remote_read> ... ]
  

# 1. 每一个抓取任务叫一个 job，通常配置里面会有多个抓取任务，比如抓取node监控，server监控等
job_name: <job_name>

# 2. 对于这个job设置的抓取间隔和超时时间 
[ scrape_interval: <duration> | default = <global_config.scrape_interval> ]
[ scrape_timeout: <duration> | default = <global_config.scrape_timeout> ]

# 3. 这个job的抓取路径
[ metrics_path: <path> | default = /metrics ]
[ scheme: <scheme> | default = http ]
params:
  [ <string>: [<string>, ...] ]
  
# 4. 其他抓取相关的配置略，主要是抓取鉴权/代理相关，配置 prometheus 如何发送抓取请求
basic_auth/ bearer_token/ bearer_token_file/ tls_config/ proxy_url


# 5. 当 抓取到的指标 label 和 服务端生成的 label产生冲突的时候如何处理，服务端生成的 label 
# 包括 job / instance/ 配置 labels, labels 和 service discovery (sd) implementations 生成的 label;
# 最后一种很重要也很容易让人迷惑，后面会具体的讲一下 sd 是如何生成和配置的
[ honor_labels: <boolean> | default = false ]
# 是否完全使用 抓取到的指标中的时间戳
[ honor_timestamps: <boolean> | default = true ]



# 6. sd 相关的配置，由于 prometheus 是主动抓取，而抓取目标往往是快速变化的，比如一个容器，他的生命周期可能很短
# 那么就存在一个如何自动发现抓取目标，已经在抓取数据上添加各种 [元Label] 的问题
# sd 就是为这个产生，根据抓取目标、服务发现机制的不同，sd 有多种实现，下面 discovery 会我们把用的比较多的
# kubernetes_sd_configs 细讲一下，其他（azure_sd_configs/consul_sd_configs/dns_sd_configs等等）略
kubernetes_sd_configs:
  [ - <kubernetes_sd_config> ... ]
  
# 这里面的主要参数就是 api_server/role/namespaces，其他鉴权相关略
# kubernetes api server 地址，不填就是 incluster config
    [ api_server: <host> ]
    # 角色 见discover一节，支持endpoints, service, pod, node, or ingress
    role: <role>
    # 抓取对象的 namespace，不填就是所有空间
    namespaces:
      names:
        [ - <string> ]


# 7. 固定的静态配置的 label
static_configs:
  [ - <static_config> ... ]

# 8. 这部分和 如何 抓取/保存 metrics 数据有关系，也比较令人困惑，下面的 scrape 一节会细讲
# 注意这里 relabel_configs，metric_relabel_configs 都是用的 relabel_config，不同的是 relabel_configs
# 还会影响如何抓取（在 scrape 之前）的动作，而 metric_relabel_configs 只会影响 抓取之后的 存储
relabel_configs:
  [ - <relabel_config> ... ]
    # 静态配置的地址
    targets:
      [ - '<host>' ]
    # Labels assigned to all metrics scraped from the targets.
    labels:
      [ <labelname>: <labelvalue> ... ]

# List of metric relabel configurations.
metric_relabel_configs:
  [ - <relabel_config> ... ]

# Per-scrape limit on number of scraped samples that will be accepted.
# If more than this number of samples are present after metric relabelling
# the entire scrape will be treated as failed. 0 means no limit.
[ sample_limit: <int> | default = 0 ]

// Run implements the discoverer interface.
func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) 

// prometheus/prometheus/discovery/targetgroup/targetgroup.go
type Group struct {
	// Targets is a list of targets identified by a label set. Each target is
	// uniquely identifiable in the group by its address label.
	Targets []model.LabelSet
	// Labels is a set of labels that is common across all targets in the group.
	Labels model.LabelSet

	// Source is an identifier that describes a group of targets.
	Source string
}


// prometheus/prometheus/discovery/kubernetes/kubernetes.go
func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
	switch d.role {
	case RoleEndpoint:
		for _, namespace := range namespaces {
		    // ....
			eps := NewEndpoints(
				log.With(d.logger, "role", "endpoint"),
				cache.NewSharedInformer(slw, &apiv1.Service{}, resyncPeriod),
				cache.NewSharedInformer(elw, &apiv1.Endpoints{}, resyncPeriod),
				cache.NewSharedInformer(plw, &apiv1.Pod{}, resyncPeriod),
			)
			d.discoverers = append(d.discoverers, eps)
			go eps.endpointsInf.Run(ctx.Done())
			go eps.serviceInf.Run(ctx.Done())
			go eps.podInf.Run(ctx.Done())
			

// prometheus/scrape/scrape.go
// 抓取数据
func (sl *scrapeLoop) run(interval, timeout time.Duration, errc chan<- error) 

=> 

// 通过 append 保存
func (sl *scrapeLoop) append(b []byte, contentType string, ts time.Time) (total, added, seriesAdded int, err error) 

=>

// 修改 label
// 1. lset是数据中的 label 
// 2. target 里面有 配置的, discover出来的 label，经过了 relabel_configs 的修改
// 3. rc 是 metric_relabel_configs，这里再次 relabel process
// 这里有个潜规则，label 修改为空了，那么表示 metrics 都不要了，整体丢弃
func mutateSampleLabels(lset labels.Labels, target *Target, honor bool, rc []*relabel.Config) labels.Labels 

# 来源 labels (label key)，多个用, 分隔 => 用于 replace, keep, and drop 
[ source_labels: '[' <labelname> [, ...] ']' ]
# label value 连接的分隔符
[ separator: <string> | default = ; ]

# 生成的 label 名，用于 replace 必填
[ target_label: <labelname> ]

# 用于提取 source label 内容的正则表达式，默认 (.*) 
[ regex: <regex> | default = (.*) ]

# hash 使用
[ modulus: <uint64> ]

# 选择 regex 使用，用于 replace
[ replacement: <string> | default = $1 ]

# Action，可用 replace；keep；drop；hashmod；labelmap；labeldrop；labelkeep
[ action: <relabel_action> | default = replace ]

// prometheus/pkg/relabel/relabel.go
func Process(labels labels.Labels, cfgs ...*Config) labels.Labels {
	for _, cfg := range cfgs {
		labels = relabel(labels, cfg)
		if labels == nil {
			return nil
		}
	}
	return labels
}

func relabel(lset labels.Labels, cfg *Config) labels.Labels {
	// 省略部分代码，val 是 label key 对应的 label value 连接
	val := strings.Join(values, cfg.Separator)
	lb := labels.NewBuilder(lset)

	switch cfg.Action {
	case Drop: if cfg.Regex.MatchString(val)  return nil
	case Keep: if !cfg.Regex.MatchString(val) return nil
	case Replace: lb[TargetLabel(val)] = Replacement(val)
	case HashMod: lb.Set(cfg.TargetLabel, fmt.Sprintf("%d", mod := sum64(md5.Sum([]byte(val))) % cfg.Modulus))
	case LabelMap: for l in lset { if cfg.Regex.MatchString(l.Name) { lb.Set(cfg.Regex.ReplaceAllString(l.Name, cfg.Replacement), l.Value) } }
	case LabelDrop: for l in lset { if cfg.Regex.MatchString(l.Name) { lb.Del(l.Name) } }
	case LabelKeep: for l in lset { if !cfg.Regex.MatchString(l.Name) { lb.Del(l.Name) } }
	}

	return lb.Labels()
}

  relabel_configs:
  - source_labels: [__meta_kubernetes_service_label_k8s_app]
    separator: ;
    regex: kube-state-metrics
    replacement: $1
    action: keep
  - source_labels: [__meta_kubernetes_endpoint_port_name]
    separator: ;
    regex: http-main
    replacement: $1
    action: keep
  - source_labels: [__meta_kubernetes_namespace]
    separator: ;
    regex: (.*)
    target_label: namespace
    replacement: $1
    action: replace

type Appender interface {
	Add(l labels.Labels, t int64, v float64) (uint64, error)

	AddFast(l labels.Labels, ref uint64, t int64, v float64) error

	// Commit submits the collected samples and purges the batch.
	Commit() error

	Rollback() error
}

// Querier provides reading access to time series data.
type Querier interface {
	// Select returns a set of series that matches the given label matchers.
	Select(*SelectParams, ...*labels.Matcher) (SeriesSet, Warnings, error)

	// LabelValues returns all potential values for a label name.
	LabelValues(name string) ([]string, Warnings, error)

	// LabelNames returns all the unique label names present in the block in sorted order.
	LabelNames() ([]string, Warnings, error)

	// Close releases the resources of the Querier.
	Close() error
}