009.ELK使用Redis做缓存收集Nginx日志

1. 流程说明

2. 配置过程

2.1 nginx配置

log_format json  '{"time_local": "$time_local", '
                          '"remote_addr": "$remote_addr", '
                          '"referer": "$http_referer", '
                          '"request": "$request", '
                          '"status": $status, '
                          '"bytes": $body_bytes_sent, '
                          '"agent": "$http_user_agent", '
                          '"x_forwarded": "$http_x_forwarded_for", '
                          '"up_addr": "$upstream_addr", '
                          '"up_host": "$upstream_http_host", '
                          '"upstream_time": "$upstream_response_time", '
                          '"request_time": "$request_time"}';
# 使用json日志格式
access_log  /var/log/nginx/access.log main;

2.2 filebeat配置

filebeat.inputs:
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]
output.redis:
  hosts: ["10.0.0.104:6379"]
  keys:
    - key: "nginx_access"   
      when.contains:
        tags: "access"
    - key: "nginx_error"
      when.contains:
        tags: "error"

2.3 logstash配置

input {
  redis {
    host => "10.0.0.104"
    port => "6379"
    db => "0"
    key => "nginx_access"
    data_type => "list"
  }
  redis {
    host => "10.0.0.104"
    port => "6379"
    db => "0"
    key => "nginx_error"
    data_type => "list"
  }
}

filter {
  mutate {
    # 这两个字段转为float类型
    convert => ["upstream_time", "float"]
    convert => ["request_time", "float"]
  }
}

output {
    stdout {}
    if "access" in [tags] {
      elasticsearch {
        hosts => ["10.0.0.101:9200"]
        manage_template => false
        index => "nginx_access-%{+yyyy.MM}"
      }
    }
    if "error" in [tags] {
      elasticsearch {
        hosts => ["10.0.0.101:9200"]
        manage_template => false
        index => "nginx_error-%{+yyyy.MM}"
      }
    }
}

3. 测试

• 启动以上服务 [root@nginx01 ~]# systemctl start nginx [root@redis01 ~]# /opt/redis/bin/redis-server /opt/redis/conf/redis_6379.conf [root@es01 ~]# systemctl start elasticsearch [root@es01 ~]# systemctl start kibana [root@nginx01 ~]# systemctl start filebeat [root@es01 ~]# /usr/share/logstash/bin/logstash -f /root/logstash.yml
• 发送测试请求 [root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/ [root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/baidu
• 查看redis 10.0.0.104:6379> keys * 1) "nginx_error" 2) "nginx_access" 10.0.0.104:6379> lpop nginx_access "{\"@timestamp\":\"2020-04-27T06:49:45.566Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"source\":\"/var/log/nginx/access.log\",\"time_local\":\"27/Apr/2020:14:49:37 +0800\",\"x_forwarded\":\"-\",\"log\":{\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"up_addr\":\"-\",\"offset\":2775,\"request\":\"GET / HTTP/1.0\",\"request_time\":\"0.000\",\"up_host\":\"-\",\"remote_addr\":\"10.0.0.109\",\"host\":{\"name\":\"nginx01\"},\"status\":200,\"referer\":\"-\",\"tags\":[\"access\"],\"prospector\":{\"type\":\"log\"},\"input\":{\"type\":\"log\"},\"beat\":{\"name\":\"nginx01\",\"hostname\":\"nginx01\",\"version\":\"6.6.0\"},\"agent\":\"ApacheBench/2.3\",\"upstream_time\":\"-\",\"bytes\":612}" 10.0.0.104:6379> lpop nginx_error "{\"@timestamp\":\"2020-04-27T06:49:55.558Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"log\":{\"file\":{\"path\":\"/var/log/nginx/error.log\"}},\"source\":\"/var/log/nginx/error.log\",\"tags\":[\"error\"],\"prospector\":{\"type\":\"log\"},\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"nginx01\"},\"message\":\"2020/04/27 14:49:47 [error] 3031#3031: *1009 open() \\\"/usr/share/nginx/html/baidu\\\" failed (2: No such file or directory), client: 10.0.0.109, server: localhost, request: \\\"GET /baidu HTTP/1.0\\\", host: \\\"10.0.0.109\\\"\",\"offset\":1040,\"beat\":{\"version\":\"6.6.0\",\"name\":\"nginx01\",\"hostname\":\"nginx01\"}}"
• logstash控制台输出 { "tags" => [ [0] "access" ], "request" => "GET /baidu HTTP/1.0", "offset" => 554720, "beat" => { "hostname" => "nginx01", "version" => "6.6.0", "name" => "nginx01" }, "referer" => "-", "time_local" => "27/Apr/2020:14:49:47 +0800", "input" => { "type" => "log" }, "host" => { "name" => "nginx01" }, "status" => 404, "up_addr" => "-", "up_host" => "-", "prospector" => { "type" => "log" }, "bytes" => 153, "@version" => "1", "agent" => "ApacheBench/2.3", "request_time" => 0.0, "upstream_time" => 0.0, "@timestamp" => 2020-04-27T06:49:48.731Z, "source" => "/var/log/nginx/access.log", "log" => { "file" => { "path" => "/var/log/nginx/access.log" } }, "x_forwarded" => "-", "remote_addr" => "10.0.0.109" }
• 查看kibana GET _cat/indices yellow open nginx_access-2020.04 hikPROoJR0OK3YiX1a-ztA 5 1 2000 0 643.2kb 643.2kb yellow open nginx_error-2020.04 QUpHiZuuQSetl0m04xrMRQ 5 1 1000 0 995.9kb 995.9kb GET nginx_access-2020.04/_search { "took" : 4, "timed_out" : false, "_shards" : { "total" : 5, "successful" : 5, "skipped" : 0, "failed" : 0 }, "hits" : { "total" : 1999, "max_score" : 1.0, "hits" : [ { "_index" : "nginx_access-2020.04", "_type" : "doc", "_id" : "avBpunEBINm9vG5xGD9v", "_score" : 1.0, "_source" : { "tags" : [ "access" ], "request" : "GET / HTTP/1.0", "offset" : 246975, "time_local" : "27/Apr/2020:14:49:37 +0800", "referer" : "-", "beat" : { "hostname" : "nginx01", "version" : "6.6.0", "name" : "nginx01" }, "input" : { "type" : "log" }, "host" : { "name" : "nginx01" }, "status" : 200, "up_addr" : "-", "up_host" : "-", "prospector" : { "type" : "log" }, "bytes" : 612, "@version" : "1", "agent" : "ApacheBench/2.3", "upstream_time" : 0.0, "request_time" : 0.0, "@timestamp" : "2020-04-27T06:49:45.660Z", "source" : "/var/log/nginx/access.log", "log" : { "file" : { "path" : "/var/log/nginx/access.log" } }, "x_forwarded" : "-", "remote_addr" : "10.0.0.109" } } ] } }