Prometheus Operator手动部署

$ git clone https://github.com/coreos/prometheus-operator.git -b release-0.29
$ cd contrib/kube-prometheus/manifests/

# 新建对应的服务目录
$ mkdir -p operator node-exporter alertmanager grafana kube-state-metrics prometheus serviceMonitor adapter

# 把对应的服务配置文件移动到相应的服务目录
$ mv *-serviceMonitor* serviceMonitor/
$ mv 0prometheus-operator* operator/
$ mv grafana-* grafana/
$ mv kube-state-metrics-* kube-state-metrics/
$ mv alertmanager-* alertmanager/
$ mv node-exporter-* node-exporter/
$ mv prometheus-adapter* adapter/
$ mv prometheus-* prometheus/

# 新创建了两个目录，存放钉钉配置和其它配置
mkdir other dingtalk-hook

$ kubectl -n monitoring get pod

NAME                                   READY     STATUS    RESTARTS   AGE
prometheus-operator-56954c76b5-qm9ww   1/1       Running   0          24s

$ kubectl get crd -n monitoring

NAME                                    CREATED AT
alertmanagers.monitoring.coreos.com     2019-04-16T06:22:20Z
prometheuses.monitoring.coreos.com      2019-04-16T06:22:20Z
prometheusrules.monitoring.coreos.com   2019-04-16T06:22:20Z
servicemonitors.monitoring.coreos.com   2019-04-16T06:22:21Z

# 把etcd证书保存到secrets中
kubectl -n monitoring create secret generic etcd-certs --from-file=/opt/kubernetes/ssl/server.pem --from-file=/opt/kubernetes/ssl/server-key.pem --from-file=/opt/kubernetes/ssl/ca.pem
 
# 加载自定义配置
cd other
kubectl create secret generic additional-configs --from-file=prometheus-additional.yaml -n monitoring
 
kubectl apply -f adapter/
kubectl apply -f alertmanager/
kubectl apply -f node-exporter/
kubectl apply -f kube-state-metrics/
kubectl apply -f grafana/
kubectl apply -f prometheus/
kubectl apply -f serviceMonitor/
 
 
# 查看是否正常部署
kubectl -n monitoring get all -o wide

# 解决方法
kubectl apply -f other/kube-controller-manager-svc-ep.yaml
kubectl apply -f other/kube-scheduler-svc-ep.yaml

# vim kube-controller-manager-svc-ep.yaml
apiVersion: v1
kind: Service
metadata:
  namespace: kube-system
  name: kube-controller-manager
  labels:
    k8s-app: kube-controller-manager
spec:
  type: ClusterIP
  clusterIP: None
  ports:
  - name: http-metrics
    port: 10252
    targetPort: 10252
    protocol: TCP

---
apiVersion: v1
kind: Endpoints
metadata:
  labels:
    k8s-app: kube-controller-manager
  name: kube-controller-manager
  namespace: kube-system
subsets:
- addresses:
  - ip: 172.16.3.9   # master ip 地址
  ports:
  - name: http-metrics
    port: 10252
    protocol: TCP
    
# vim kube-scheduler-svc-ep.yaml
apiVersion: v1
kind: Service
metadata:
  namespace: kube-system
  name: kube-scheduler
  labels:
    k8s-app: kube-scheduler
spec:
  type: ClusterIP
  clusterIP: None
  ports:
  - name: http-metrics
    port: 10251
    targetPort: 10251
    protocol: TCP

---
apiVersion: v1
kind: Endpoints
metadata:
  labels:
    k8s-app: kube-scheduler
  name: kube-scheduler
  namespace: kube-system
subsets:
- addresses:
  - ip: 172.16.3.9 # master ip 地址
  ports:
  - name: http-metrics
    port: 10251
    protocol: TCP

# 解决方法 添加监控端口 9153

$ kubectl edit svc -n kube-system kube-dns
 
  - name: metrics
    port: 9153
    protocol: TCP
    targetPort: 9153

# 创建配置
$ vim prometheus-additional.yaml

- job_name: 'crm'
  scrape_interval: 5s
  metrics_path: '/api/private/metrics'
  static_configs:
    - targets: ['www.example.com']

# 加载配置
$ kubectl create secret generic additional-configs --from-file=prometheus-additional.yaml -n monitoring

# 钉钉推送消息安装配置方法
https://github.com/yangpeng14/prometheus-operator-configure/tree/master/dingtalk-hook