OpenResty + Lua + Kafka 实现日志收集系统以及部署过程中遇到的坑

server {  
    listen       80;  
    server_name  localhost;  
    location /favicon.ico {  
        root   html;  
            index  index.html index.htm;  
    }  
    location / {  
        proxy_connect_timeout 8;  
        proxy_send_timeout 8;  
        proxy_read_timeout 8;  
        proxy_buffer_size 4k;  
        proxy_buffers 512 8k;  
        proxy_busy_buffers_size 8k;  
        proxy_temp_file_write_size 64k;  
        proxy_next_upstream http_500 http_502  http_503 http_504  error timeout invalid_header;  
        root   html;  
        index  index.html index.htm;  
        proxy_pass http://rc;  
        proxy_http_version 1.1;  
        proxy_set_header Connection "";  
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;  
        # 使用log_by_lua 包含lua代码,因为log_by_lua指令运行在请求最后且不影响proxy_pass机制  
        log_by_lua '  
            -- 引入lua所有api  
            local cjson = require "cjson"  
            local producer = require "resty.kafka.producer"  
            -- 定义kafka broker地址，ip需要和kafka的host.name配置一致  
            local broker_list = {  
                { host = "10.10.78.52", port = 9092 },  
            }  
            -- 定义json便于日志数据整理收集  
            local log_json = {}  
            log_json["uri"]=ngx.var.uri  
            log_json["args"]=ngx.var.args  
            log_json["host"]=ngx.var.host  
            log_json["request_body"]=ngx.var.request_body  
            log_json["remote_addr"] = ngx.var.remote_addr  
            log_json["remote_user"] = ngx.var.remote_user  
            log_json["time_local"] = ngx.var.time_local  
            log_json["status"] = ngx.var.status  
            log_json["body_bytes_sent"] = ngx.var.body_bytes_sent  
            log_json["http_referer"] = ngx.var.http_referer  
            log_json["http_user_agent"] = ngx.var.http_user_agent  
            log_json["http_x_forwarded_for"] = ngx.var.http_x_forwarded_for  
            log_json["upstream_response_time"] = ngx.var.upstream_response_time  
            log_json["request_time"] = ngx.var.request_time  
            -- 转换json为字符串  
            local message = cjson.encode(log_json);  
            -- 定义kafka异步生产者  
            local bp = producer:new(broker_list, { producer_type = "async" })  
            -- 发送日志消息,send第二个参数key,用于kafka路由控制:  
            -- key为nill(空)时，一段时间向同一partition写入数据  
            -- 指定key，按照key的hash写入到对应的partition  
            local ok, err = bp:send("test1", nil, message)  

            if not ok then  
                ngx.log(ngx.ERR, "kafka send err:", err)  
                return  
            end  
        ';  
    }  
    error_page   500 502 503 504  /50x.html;  
    location = /50x.html {  
        root   html;  
    }  
}  

#user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    upstream myServer {
    server 192.168.0.109:8080 weight=1;
    }

    lua_package_path "/opt/openresty/lualib/kafka/?.lua;;";
    lua_need_request_body on;

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location /test1 {
       # 请求转向自定义的服务器列表
            proxy_pass http://myServer;
        }

    location /test2 {

        # 使用log_by_lua 包含lua代码,因为log_by_lua指令运行在请求最后且不影响proxy_pass机制  
        log_by_lua '  
            -- 引入lua所有api
        local topic = "test"
            local cjson = require "cjson"  
            local producer = require "resty.kafka.producer"  
            -- 定义kafka broker地址，ip需要和kafka的host.name配置一致  
            local broker_list = {  
                { host = "192.168.0.109", port = 9092 },
        { host = "192.168.0.110", port = 9092 },
        { host = "192.168.0.101", port = 9092 }
            }  
            -- 定义json便于日志数据整理收集  
            local log_json = {}  
            log_json["uri"]=ngx.var.uri  
            log_json["args"]=ngx.req.get_uri_args()  
            log_json["host"]=ngx.var.host  
            log_json["request_body"]=ngx.var.request_body  
            log_json["remote_addr"] = ngx.var.remote_addr  
            log_json["remote_user"] = ngx.var.remote_user  
            log_json["time_local"] = ngx.var.time_local  
            log_json["status"] = ngx.var.status  
            log_json["body_bytes_sent"] = ngx.var.body_bytes_sent  
            log_json["http_referer"] = ngx.var.http_referer  
            log_json["http_user_agent"] = ngx.var.http_user_agent  
            log_json["http_x_forwarded_for"] = ngx.var.http_x_forwarded_for  
            log_json["upstream_response_time"] = ngx.var.upstream_response_time  
            log_json["request_time"] = ngx.var.request_time  
            -- 转换json为字符串  
            local message = cjson.encode(ngx.req.get_uri_args());  
            -- 定义kafka异步生产者  
            local bp = producer:new(broker_list, { producer_type = "async" })  
            -- 发送日志消息,send第二个参数key,用于kafka路由控制:  
            -- key为nill(空)时，一段时间向同一partition写入数据  
            -- 指定key，按照key的hash写入到对应的partition  
            local ok, err = bp:send(topic, nil, message)  

            if not ok then  
                ngx.log(ngx.ERR, "kafka send err:", err)  
                return  
            end  
        ';  
        }  


        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }


    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}


    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}