log_format json '{"time_local": "$time_local", '
'"remote_addr": "$remote_addr", '
'"referer": "$http_referer", '
'"request": "$request", '
'"status": $status, '
'"bytes": $body_bytes_sent, '
'"agent": "$http_user_agent", '
'"x_forwarded": "$http_x_forwarded_for", '
'"up_addr": "$upstream_addr", '
'"up_host": "$upstream_http_host", '
'"upstream_time": "$upstream_response_time", '
'"request_time": "$request_time"}';
# 使用json日志格式
access_log /var/log/nginx/access.log main;
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["access"]
- type: log
enabled: true
paths:
- /var/log/nginx/error.log
tags: ["error"]
output.kafka:
hosts: ["10.0.0.110:9092","10.0.0.111:9092","10.0.0.112:9092"]
topic: nginx_log
input {
kafka {
bootstrap_servers => "10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092"
topics => ["nginx_log"]
group_id => "logstash"
codec => "json"
}
}
filter {
mutate {
convert => ["upstream_time", "float"]
convert => ["request_time", "float"]
}
}
output {
if "access" in [tags] {
elasticsearch {
hosts => "http://10.0.0.101:9200"
manage_template => false
index => "nginx_access-%{+yyyy.MM}"
}
}
if "error" in [tags] {
elasticsearch {
hosts => "http://10.0.0.101:9200"
manage_template => false
index => "nginx_error-%{+yyyy.MM}"
}
}
}
[root@kafka01 ~]# /opt/kafka/bin/kafka-topics.sh --create --bootstrap-server
10.0.0.110:9092,10.0.0.111:9092,10.0.0.111:9092 --replication-factor 3 --partitions 3 --topic nginx_log
[root@kafka03 ~]# /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server
10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092 --topic nginx_log --from-beginning
[root@nginx01 ~]# systemctl start nginx [root@es01 ~]# systemctl start elasticsearch
[root@es01 ~]# systemctl start kibana [root@nginx01 ~]# systemctl start filebeat [root@es01 ~]# /usr/share/logstash/bin/logstash -f /root/logstash.yml
[root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/ [root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/baidu
[root@kafka03 ~]# /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server
10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092
--topic nginx_log --from-beginning
{"@timestamp":"2020-04-27T09:09:34.585Z","@metadata":{
"beat":"filebeat","type":"doc","version":"6.6.0","topic":"nginx_log"},
"beat":{"name":"nginx01","hostname":"nginx01","version":"6.6.0"},
"x_forwarded":"-","source":"/var/log/nginx/access.log","bytes":153,
"request":"GET /baidu HTTP/1.0","status":404,"offset":552760,
"up_host":"-","input":{"type":"log"},"time_local":"27/Apr/2020:17:09:34 +0800",
"tags":["access"],"host":{"name":"nginx01"},"log":{"file":{"path":"/var/log/nginx/access.log"}},
"up_addr":"-","remote_addr":"10.0.0.109","request_time":"0.000",
"upstream_time":"-","referer":"-","agent":"ApacheBench/2.3","prospector":{"type":"log"}}
{"@timestamp":"2020-04-27T09:09:34.585Z","@metadata":{"beat":"filebeat",
"type":"doc","version":"6.6.0","topic":"nginx_log"},
"agent":"ApacheBench/2.3","request_time":"0.000","prospector":{"type":"log"},
"beat":{"version":"6.6.0","name":"nginx01","hostname":"nginx01"},"host":{"name":"nginx01"},
"status":404,"up_host":"-","remote_addr":"10.0.0.109","tags":["access"],
"source":"/var/log/nginx/access.log","log":{"file":{"path":"/var/log/nginx/access.log"}},
"up_addr":"-","referer":"-","request":"GET /baidu HTTP/1.0","x_forwarded":"-","bytes":153,
"time_local":"27/Apr/2020:17:09:34 +0800","input":{"type":"log"},"offset":553880,"upstream_time":"-"} ......
GET _cat/indices yellow open nginx_access-2020.04
apimPU-QTAmP7GeE7l8evQ 5 1 2000 0 689kb 689kb yellow open nginx_error-2020.04
WH3Lme3gQuqBSK8MFYfsSw 5 1 1000 0 754.2kb 754.2kb GET nginx_access-2020.04/_search
{ "took" : 4, "timed_out" : false, "_shards" : { "total" : 5, "successful" : 5,
"skipped" : 0, "failed" : 0 }, "hits" : { "total" : 1999, "max_score" : 1.0,
"hits" : [ { "_index" : "nginx_access-2020.04", "_type" : "doc",
"_id" : "avBpunEBINm9vG5xGD9v", "_score" : 1.0, "_source" : {
"tags" : [ "access" ], "request" : "GET / HTTP/1.0",
"offset" : 246975, "time_local" : "27/Apr/2020:14:49:37 +0800",
"referer" : "-", "beat" : { "hostname" : "nginx01",
"version" : "6.6.0", "name" : "nginx01" }, "input" : {
"type" : "log" }, "host" : { "name" : "nginx01" },
"status" : 200, "up_addr" : "-", "up_host" : "-",
"prospector" : { "type" : "log" }, "bytes" : 612,
"@version" : "1", "agent" : "ApacheBench/2.3",
"upstream_time" : 0.0, "request_time" : 0.0, "@timestamp" :
"2020-04-27T06:49:45.660Z", "source" : "/var/log/nginx/access.log",
"log" : { "file" : { "path" : "/var/log/nginx/access.log" }
}, "x_forwarded" : "-", "remote_addr" : "10.0.0.109" } } ] } }