使用Prometheus实现大规模的应用程序监视【Containers】

import java.io.IOException;
import io.prometheus.client.Counter;
import io.prometheus.client.Gauge;
import io.prometheus.client.exporter.HTTPServer;
import io.prometheus.client.hotspot.DefaultExports;
 
// https://github.com/prometheus/client_java
// Demo of how we plan to use Prometheus Java client to instrument Anomalia Machina.
// Note that the Anomalia Machina application will have Kafka Producer and Kafka consumer and rest of pipeline running in multiple separate processes/instances.
// So metrics from each will have different host/port combinations.
public class PrometheusBlog {  
static String appName = "prometheusTest";
// counters can only increase in value (until process restart)
// Execution count. Use a single Counter for all stages of the pipeline, stages are distinguished by labels
static final Counter pipelineCounter = Counter.build()
    .name(appName + "_requests_total").help("Count of executions of pipeline stages")
    .labelNames("stage")
    .register();
// in theory could also use pipelineCounter to count anomalies found using another label
// but less potential for confusion having another counter. Doesn't need a label
static final Counter anomalyCounter = Counter.build()
    .name(appName + "_anomalies_total").help("Count of anomalies detected")
    .register();
// A Gauge can go up and down, and is used to measure current value of some variable.
// pipelineGauge will measure duration in seconds of each stage using labels.
static final Gauge pipelineGauge = Gauge.build()
    .name(appName + "_duration_seconds").help("Gauge of stage durations in seconds")
    .labelNames("stage")
    .register();
 
public static void main(String[] args) {
// Allow default JVM metrics to be exported
   DefaultExports.initialize();
 
   // Metrics are pulled by Prometheus, create an HTTP server as the endpoint
   // Note if there are multiple processes running on the same server need to change port number.
   // And add all IPs and port numbers to the Prometheus configuration file.
HTTPServer server = null;
try {
server = new HTTPServer(1234);
} catch (IOException e) {
e.printStackTrace();
}
// now run 1000 executions of the complete pipeline with random time delays and increasing rate
int max = 1000;
for (int i=0; i < max; i++)
{
// total time for complete pipeline, and increment anomalyCounter
pipelineGauge.labels("total").setToTime(() -> {
producer();
consumer();
if (detector())
anomalyCounter.inc();
});
// total pipeline count
pipelineCounter.labels("total").inc();
System.out.println("i=" + i);
 
// increase the rate of execution
try {
Thread.sleep(max-i);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
server.stop();
}
// the 3 stages of the pipeline, for each we increase the stage counter and set the Gauge duration time
public  static void producer() {
class Local {};
String name = Local.class.getEnclosingMethod().getName();
pipelineGauge.labels(name).setToTime(() -> {
try {
Thread.sleep(1 + (long)(Math.random()*20));
} catch (InterruptedException e) {
e.printStackTrace();
}
});
pipelineCounter.labels(name).inc();
   }
public  static void consumer() {
class Local {};
String name = Local.class.getEnclosingMethod().getName();
pipelineGauge.labels(name).setToTime(() -> {
try {
Thread.sleep(1 + (long)(Math.random()*10));
} catch (InterruptedException e) {
e.printStackTrace();
}
});
pipelineCounter.labels(name).inc();
   }
// detector returns true if anomaly detected else false
public  static boolean detector() {
class Local {};
String name = Local.class.getEnclosingMethod().getName();
pipelineGauge.labels(name).setToTime(() -> {
try {
Thread.sleep(1 + (long)(Math.random()*200));
} catch (InterruptedException e) {
e.printStackTrace();
}
});
pipelineCounter.labels(name).inc();
return (Math.random() > 0.95);
   }
}

<!-- The client -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
<version>LATEST</version>
</dependency>
<!-- Hotspot JVM metrics-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_hotspot</artifactId>
<version>LATEST</version>
</dependency>
<!-- Exposition HTTPServer-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_httpserver</artifactId>
<version>LATEST</version>
</dependency>
<!-- Pushgateway exposition-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_pushgateway</artifactId>
<version>LATEST</version>
</dependency>

global:
 scrape_interval:    15s # By default, scrape targets every 15 seconds.
 
# scrape_configs has jobs and targets to scrape for each.
scrape_configs:
# job 1 is for testing prometheus instrumentation from multiple application processes.
 # The job name is added as a label job=<job_name> to any timeseries scraped from this config.
 - job_name: 'testprometheus'
 
   # Override the global default and scrape targets from this job every 5 seconds.
   scrape_interval: 5s
   
   # this is where to put multiple targets, e.g. for Kafka load generators and detectors
   static_configs:
     - targets: ['localhost:1234', 'localhost:1235']
     
 # job 2 provides operating system metrics (e.g. CPU, memory etc).
 - job_name: 'node'
 
  # Override the global default and scrape targets from this job every 5 seconds.
   scrape_interval: 5s
   
   static_configs:
     - targets: ['localhost:9100']