对于ELK部署使用而言,下面是一个再常见不过的架构了
[root@localhost ~]# cat /etc/redhat-release
CentOS Linux release 7.3.1611 (Core)
wget https://download.elastic.co/logstash/logstash/logstash-2.3.2.tar.gz
wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.2/elasticsearch-2.3.2.tar.gz
wget https://download.elastic.co/kibana/kibana/kibana-4.5.0-linux-x64.tar.gz
wget https://github.com/antirez/redis/archive/3.2.0.tar.gz
#安装nginx
yum install nginx -y
#启动nginx
nginx
#查看端口
netstat -nplt | grep nginx
#修改nginx访问日志
[root@localhost ~]# vi /etc/nginx/nginx.conf
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
[root@localhost ~]# vi /etc/nginx/conf.d/default.conf
#定义nginx日志使用的格式,以及日志文件的位置
access_log /var/log/nginx/access.log main;
#重启nginx
nginx -s reload
#安装依赖
yum install tcl -y
yum groupinstall "Development Tools" "Compatibility Libraries" -y
#安装redis
tar xvf 3.2.0.tar.gz -C /usr/local
cd /usr/local
mv redis-3.2.0 redis
cd redis
make
make test
make install
mkdir /etc/redis
cp redis/redis.conf /etc/redis
#配置redis
[root@localhost ~]# vim /etc/redis/redis.conf
daemonize yes
启动redis
[root@localhost ~]# redis-server /etc/redis/redis.conf
[root@localhost ~]# netstat -nltp | grep redis
tcp 0 0 127.0.0.1:6379 0.0.0.0:* LISTEN 32357/redis-server
安装java
yum install java-1.8.0-openjdk -y
安装Logstash
tar xvf logstash-2.3.2.tar.gz -C /usr/local/
#在终端中,像下面这样运行命令来启动 Logstash 进程:
$ /usr/local/logstash-2.3.2/bin/logstash -e 'input{stdin{}}output{stdout{codec=>rubydebug}}'
hello world
$ /usr/local/logstash-2.3.2/bin/logstash -e 'input{stdin{}}output{stdout{codec=>rubydebug}}'
hello world
执行完命令,然后你会发现终端在等待你的输入。没问题,敲入hello world,然后回车,logstash会返回以下结果!
{
"message" => "hello world",
"@version" => "1",
"@timestamp" => "2017-11-04T11:50:27.036Z",
"host" => "localhost.localdomain"
}
编写Shipper角色的配置文件:logstash_shipper.conf
$ cat /usr/local/logstash-2.3.2/etc/logstash_shipper.conf
input {
file {
type => "nginx_access log" #这里定义的是日志文件名;
path => ["/var/log/nginx/access.log"] #这里定义的是日志文件路径;
}
}
output {
redis {
host => "localhost" #redis主机地址,这里是本机;
port => 6379 #redis端口号;
data_type => "list" #使用redis队列模式;
key => "logstash:redis" #队列通道的名称;
}
}
编写indexer角色的配置文件:logstash_indexer.conf
cat /usr/local/logstash-2.3.2/etc/logstash_indexer.conf
input {
redis { #去redis队列取数据;
host => "localhost" #连接redis服务器;
port => 6379 #连接redis端口;
data_type => "list" #数据类型;
key => "logstash:redis" #队列名称;
}
}
output {
elasticsearch { #Logstash输出到elasticsearch;
hosts => ["localhost"] #elasticsearch为本地;
index => "logstash-nginx-%{+YYYY.MM.dd}" #创建索引;
document_type => "nginx" #文档类型;
workers => 1 #进程数量;
flush_size => 20000
idle_flush_time => 10
}
}
#启动
nohup /usr/local/logstash-2.3.2/bin/logstash -f /usr/local/logstash-2.3.2/etc/logstash_shipper.conf &
nohup /usr/local/logstash-2.3.2/bin/logstash -f /usr/local/logstash-2.3.2/etc/logstash_indexer.conf
tar xvf elasticsearch-2.3.2.tar.gz -C /usr/local
#创建用户
useradd elk
chown elk.elk -R /usr/local/elastcsearch.2.3.2
#修改elasticsearch配置文件
cat /usr/local/elasticsearch-2.3.2/config/elasticsearch.yml
cluster.name: elasticsearch
node.name: node1
node.box_type: stale
path.data: ['/data/elasticsearch']
path.logs: /var/log/elasticsearch/
network.host: 0.0.0.0
index.number_of_replicas: 0
#创建elasticsearch需要的数据目录和日志目录。
mkdir -p /data/elasticsearch
mkdir -p /var/log/elasticsearch/
chown elk.elk /data/elasticsearch/ -R
chown elk.elk /var/log/elasticsearch/ -R
#启动
nohup sudo -u elk /usr/local/elasticsearch-2.3.2/bin/elasticsearch &
#访问
[root@localhost ~]# curl http://128.0.0.71:9200/
{
"name" : "node1",
"cluster_name" : "elasticsearch",
"version" : {
"number" : "2.3.2",
"build_hash" : "b9e4a6acad4008027e4038f6abed7f7dba346f94",
"build_timestamp" : "2016-04-21T16:03:47Z",
"build_snapshot" : false,
"lucene_version" : "5.5.0"
},
"tagline" : "You Know, for Search"
}
/usr/local/elasticsearch-2.3.2/bin/plugin install mobz/elasticsearch-head
查看插件
/usr/local/elasticsearch-2.3.2/bin/plugin list
tar xvf kibana-4.5.0-linux-x64.tar.gz -C /usr/local/
启动
nohup /usr/local/kibana-4.5.0-linux-x64/bin/kibana &