efk收集docker nginx日志

johnhuster的分享

发布于 2022-03-28 20:10:54

6170

发布于 2022-03-28 20:10:54

文章被收录于专栏：johnhuster

1、部署efk

efk需要用到elasticsearch、fluentd以及kibana，nginx使用fluentd日志驱动将nginx docker日志转发到对应fluentd server端，fluentd server端将日志加工后传递到elasticsearch，存储到elasticsearch的数据就可以使用kibana展示出来。

2、部署ek环境，

version: "3"
services:
  kibana:
    image: docker.io/kibana:6.8.0
    environment:
      - ELASTICSEARCH_URL=http://elasticsearch:9200
    container_name: kibana680
    hostname: kibana
    depends_on:
      - elasticsearch
    restart: always
    ports:
      - "5601:5601"

  elasticsearch:
    image: docker.io/elasticsearch:6.8.0
    environment:
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    volumes:
      - ./es/data:/usr/share/elasticsearch/data
    container_name: elasticsearch
    hostname: elasticsearch
    restart: always
    privileged: true
    restart: always
    ports:
      - "9200:9200"
      - "9300:9300"

使用docker-compose up -d启动elasticsearch以及kibana服务

3、部署fluentd环境

version: "3"
services:
  fluentd:
    image: fluent_fluentd
    container_name: fluentd
    volumes:
      - ./fluentd/conf:/fluentd/etc
    privileged: true
    ports:
      - "24224:24224"
    environment:
      - TZ=Asia/Shanghai
    restart: always
    logging:
        driver: "json-file"
        options:
            max-size: 100m
            max-file: "5"

下面给出nginx相关的配置部分

<source>
  @type forward
  port 24224
  bind 0.0.0.0
</source>
<filter nginx>
  @type parser
  key_name log
  <parse>
        @type regexp
        expression (?<remote>[^ ]*) (?<user>[^ ]*) \[(?<localTime>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) (?<requestTime>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+(?<http_x_forwarded_for>[^ ]+))?)?
        time_format %d/%b/%Y:%H:%M:%S %z
  </parse>
</filter>
<match nginx>
  @type copy
  <store>
    @type elasticsearch
    host 172.21.48.48
    port 9200
    logstash_format true
    logstash_prefix nginx
    logstash_dateformat %Y%m%d
    include_tag_key true
    type_name access_log
    flush_interval 1s
    include_tag_key true
    tag_key @log_name
  </store>
  <store>
    @type stdout
  </store>
</match>

fluent_fluentd是笔者自己构建的镜像，具体参考https://blog.csdn.net/john1337/article/details/102665725

注意上面的正则表达式对应的nginx日志格式为：

    log_format  main  '$remote_addr $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent $request_time "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

4、部署nginx

services:
  nginx:
    restart: always
    image: nginx
    container_name: nginx
    ports:
      - 8081:80
      - 443:443
      - 8084:8084
    volumes:
      - ./conf/nginx.conf:/etc/nginx/nginx.conf
      - ./conf.d:/etc/nginx/conf.d
      - ./www:/usr/share/nginx/html
      #- ./log:/var/log/nginx   #一定不要把nginx docker日志挂载到外部，否则fluentd无法正常工作
    privileged: true
    environment:
      - TZ=Asia/Shanghai
    logging:
        driver: "fluentd"    #日志驱动换成fluentd，默认为json-file
        options:
          fluentd-address: xx.xx.xx.xx:24224  #对应fluentd服务地址
          fluentd-async-connect: 'true'
          tag: nginx

上述为nginx部署yml文件，使用docker-compose up -d启动nginx服务即可