011.ELK使用Kafka做缓存收集Nginx日志

log_format json  '{"time_local": "$time_local", '
                          '"remote_addr": "$remote_addr", '
                          '"referer": "$http_referer", '
                          '"request": "$request", '
                          '"status": $status, '
                          '"bytes": $body_bytes_sent, '
                          '"agent": "$http_user_agent", '
                          '"x_forwarded": "$http_x_forwarded_for", '
                          '"up_addr": "$upstream_addr", '
                          '"up_host": "$upstream_http_host", '
                          '"upstream_time": "$upstream_response_time", '
                          '"request_time": "$request_time"}';
# 使用json日志格式
access_log  /var/log/nginx/access.log main;

filebeat.inputs:
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]
output.kafka:
  hosts: ["10.0.0.110:9092","10.0.0.111:9092","10.0.0.112:9092"]
  topic: nginx_log

input {
  kafka {
    bootstrap_servers => "10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092"
    topics => ["nginx_log"]
    group_id => "logstash"
    codec => "json"
  }
}

filter {
  mutate {
    convert => ["upstream_time", "float"]
    convert => ["request_time", "float"]
  }
}

output {
    if "access" in [tags] {
      elasticsearch {
        hosts => "http://10.0.0.101:9200"
        manage_template => false
        index => "nginx_access-%{+yyyy.MM}"
      }
    }
    if "error" in [tags] {
      elasticsearch {
        hosts => "http://10.0.0.101:9200"
        manage_template => false
        index => "nginx_error-%{+yyyy.MM}"
      }
    }
}

 [root@kafka01 ~]# /opt/kafka/bin/kafka-topics.sh --create --bootstrap-server 
10.0.0.110:9092,10.0.0.111:9092,10.0.0.111:9092 --replication-factor 3 --partitions 3 --topic nginx_log
 

 [root@kafka03 ~]# /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server 
10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092 --topic nginx_log --from-beginning
 

 [root@nginx01 ~]# systemctl start nginx [root@es01 ~]# systemctl start elasticsearch 
[root@es01 ~]# systemctl start kibana [root@nginx01 ~]# systemctl start filebeat [root@es01 ~]# /usr/share/logstash/bin/logstash -f /root/logstash.yml
 

 [root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/ [root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/baidu
 

 [root@kafka03 ~]# /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server 
10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092 
 --topic nginx_log --from-beginning 
{"@timestamp":"2020-04-27T09:09:34.585Z","@metadata":{
 "beat":"filebeat","type":"doc","version":"6.6.0","topic":"nginx_log"},
"beat":{"name":"nginx01","hostname":"nginx01","version":"6.6.0"},
 "x_forwarded":"-","source":"/var/log/nginx/access.log","bytes":153,
"request":"GET /baidu HTTP/1.0","status":404,"offset":552760,
 "up_host":"-","input":{"type":"log"},"time_local":"27/Apr/2020:17:09:34 +0800",
"tags":["access"],"host":{"name":"nginx01"},"log":{"file":{"path":"/var/log/nginx/access.log"}},
 "up_addr":"-","remote_addr":"10.0.0.109","request_time":"0.000",
"upstream_time":"-","referer":"-","agent":"ApacheBench/2.3","prospector":{"type":"log"}} 
 {"@timestamp":"2020-04-27T09:09:34.585Z","@metadata":{"beat":"filebeat",
"type":"doc","version":"6.6.0","topic":"nginx_log"},
"agent":"ApacheBench/2.3","request_time":"0.000","prospector":{"type":"log"},
"beat":{"version":"6.6.0","name":"nginx01","hostname":"nginx01"},"host":{"name":"nginx01"},
"status":404,"up_host":"-","remote_addr":"10.0.0.109","tags":["access"],
"source":"/var/log/nginx/access.log","log":{"file":{"path":"/var/log/nginx/access.log"}},
"up_addr":"-","referer":"-","request":"GET /baidu HTTP/1.0","x_forwarded":"-","bytes":153,
"time_local":"27/Apr/2020:17:09:34 +0800","input":{"type":"log"},"offset":553880,"upstream_time":"-"} ......
 

 GET _cat/indices  yellow open nginx_access-2020.04            
apimPU-QTAmP7GeE7l8evQ 5 1 2000  0   689kb   689kb yellow open nginx_error-2020.04             
 WH3Lme3gQuqBSK8MFYfsSw 5 1 1000  0 754.2kb 754.2kb  GET nginx_access-2020.04/_search  
{   "took" : 4,   "timed_out" : false,   "_shards" : {     "total" : 5,     "successful" : 5,     
 "skipped" : 0,     "failed" : 0   },   "hits" : {     "total" : 1999,     "max_score" : 1.0,     
"hits" : [       {         "_index" : "nginx_access-2020.04",         "_type" : "doc",         
 "_id" : "avBpunEBINm9vG5xGD9v",         "_score" : 1.0,         "_source" : {           
"tags" : [             "access"           ],           "request" : "GET / HTTP/1.0",          
  "offset" : 246975,           "time_local" : "27/Apr/2020:14:49:37 +0800",           
"referer" : "-",           "beat" : {             "hostname" : "nginx01",             
  "version" : "6.6.0",             "name" : "nginx01"           },           "input" : {            
 "type" : "log"           },           "host" : {             "name" : "nginx01"           },     
        "status" : 200,           "up_addr" : "-",           "up_host" : "-",          
  "prospector" : {             "type" : "log"           },           "bytes" : 612,          
         "@version" : "1",           "agent" : "ApacheBench/2.3",          
   "upstream_time" : 0.0,           "request_time" : 0.0,           "@timestamp" :
          "2020-04-27T06:49:45.660Z",           "source" : "/var/log/nginx/access.log",          
    "log" : {             "file" : {               "path" : "/var/log/nginx/access.log"             }
                     },           "x_forwarded" : "-",           "remote_addr" : "10.0.0.109"         }       }     ]   } }