Logstash收集Nginx访问日志
安装nginx
#直接yum安装： [root@elk-node1 ~]# yum install nginx -y 官方文档：http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format #修改配置文件的日志格式： vim /etc/nginx/nginx.conf #在http模块中添加 log_format json '{"@timestamp":"$time_iso8601",' '"@version":"1",' '"client":"$remote_addr",' '"url":"$uri",' '"status":"$status",' '"domain":"$host",' '"host":"$server_addr",' '"size":$body_bytes_sent,' '"responsetime":$request_time,' '"referer": "$http_referer",' '"ua": "$http_user_agent"' '}'; #在server模块中添加 access_log /var/log/nginx/access_json.log json; #修改后的nginx.conf文件 [root@elk-node1 ~]# grep -Ev "#|^&" /etc/nginx/nginx.conf user nginx; worker_processes auto; error_log /var/log/nginx/error.log; pid /run/nginx.pid; include /usr/share/nginx/modules/*.conf; events { worker_connections 1024; } http { log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; log_format json '{"@timestamp":"$time_iso8601",' '"@version":"1",' '"client":"$remote_addr",' '"url":"$uri",' '"status":"$status",' '"domain":"$host",' '"host":"$server_addr",' '"size":$body_bytes_sent,' '"responsetime":$request_time,' '"referer": "$http_referer",' '"ua": "$http_user_agent"' '}'; sendfile on; tcp_nopush on; tcp_nodelay on; keepalive_timeout 65; types_hash_max_size 2048; include /etc/nginx/mime.types; default_type application/octet-stream; include /etc/nginx/conf.d/*.conf; server { listen 80 default_server; listen [::]:80 default_server; server_name _; root /usr/share/nginx/html; include /etc/nginx/default.d/*.conf; access_log /var/log/nginx/access_json.log json; location / { } error_page 404 /404.html; location = /40x.html { } error_page 500 502 503 504 /50x.html; location = /50x.html { } } } #启动： [root@controller ~]# systemctl start nginx root@elk-node1 ~]# ss -lntp|grep 80 LISTEN 0 511 *:80 *:* users:(("nginx",pid=8045,fd=6),("nginx",pid=8044,fd=6),("nginx",pid=8043,fd=6)) LISTEN 0 511 :::80 :::* users:(("nginx",pid=8045,fd=7),("nginx",pid=8044,fd=7),("nginx",pid=8043,fd=7))
浏览器访问：http://192.168.247.135/
查看nginx日志
编写logstash
#添加nginx日志格式到之前logstash的elk-log.yml [root@elk-node1 ~]# cat /etc/logstash/conf.d/elk_log.conf input { file { path => "/var/log/messages" type => "system" start_position => "beginning" } file { path => "/var/log/elasticsearch/hejianlai.log" type => "es-error" start_position => "beginning" codec => multiline { pattern => "^\[" negate => true what => "previous" } } file { path => "/var/log/nginx/access_json.log" codec => json start_position => "beginning" type => "nginx-log" } } output { if [type] == "system"{ elasticsearch { hosts => ["192.168.247.135:9200"] index => "systemlog-%{+YYYY.MM.dd}" } } if [type] == "es-error"{ elasticsearch { hosts => ["192.168.247.135:9200"] index => "es-error-%{+YYYY.MM.dd}" } } if [type] == "nginx-log"{ elasticsearch { hosts => ["192.168.247.135:9200"] index => "nginx-log-%{+YYYY.MM.dd}" } } }
#添加--configtest参数检查配置语法是否有误！！！ [root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf --configtest Configuration OK #把之前后台运行的进程kill掉重启： [root@elk-node1 ~]# ps aux|grep elk root 3248 0.8 6.0 3632960 234924 pts/2 Sl 11:25 1:10 /usr/local/java/jdk1.8.0_171/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xss2048k -Djffi.boot.library.path=/opt/logstash/vendor/jruby/lib/jni -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/logstash/heapdump.hprof -Xbootclasspath/a:/opt/logstash/vendor/jruby/lib/jruby.jar -classpath ::/usr/local/java/jdk1.8.0_171/lib:/usr/local/java/jdk1.8.0_171/jre/lib -Djruby.home=/opt/logstash/vendor/jruby -Djruby.lib=/opt/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main --1.9 /opt/logstash/lib/bootstrap/environment.rb logstash/runner.rb agent -f /etc/logstash/conf.d/elk_log.conf root 8135 0.0 0.0 112704 976 pts/0 S+ 13:38 0:00 grep --color=auto elk [root@elk-node1 ~]# kill -9 3248 You have new mail in /var/spool/mail/root [root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf & [1] 8178
kibana添加nginx日志
首先在es插件中我们能看到nginx-log的索引
设置kibana
