elk 分析nginx访问和错误日志

1 nginx 日志格式配置

[root@elk-5-10 config]# cd /usr/local/nginx/conf/

[root@elk-5-10 conf]# vi nginx.conf

log_format  access  '$http_host $remote_addr - $remote_user [$time_local] "$request" '

             '$status $body_bytes_sent "$http_referer" '

             '"$http_user_agent" $http_x_forwarded_for';

2 日志格式数据样品

2.1 访问日志：

ss00.xxxxxx.me 150.138.154.157 - - [25/Jul/2017:03:02:35 +0800] "GET /csm/7_527.html HTTP/1.1" 304 0 "http://www.twww.com/tetris/page/64000159042/?ad_id=62928537191&cid=62928889880&req_id=0" "Mozilla/5.0 (Linux; Android 6.0.1;
Redmi 4X Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36" 183.204.183.118

2.2 错误日志

2017/07/25 15:23:29 [error] 24881#0: *33 open() "/usr/local/nginx-1.12.0/html/favicon.ico" failed (2: No such file or directory), client: 192.168.1.103, server: www.zyb.com, request: "GET /favicon.ico HTTP/1.1", host: "www.zyb.com"

3 logstash 配置文件

input {

    file {

        type => "nginx-access"

        path => "/data/weixin.sys.mingyaohui.com.log"

        start_position => beginning  

    }

    file {

        type => "nginx-error"

        path => "/data/nginx_error.log"

        start_position => beginning

    }

}

filter {

    if [type] == "nginx-access" {

        grok {

            match => ["message","%{IPORHOST:clientip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer})
%{QS:agent} %{IPORHOST:forwordip}" ]}

        }

    } else if [type] == "nginx-error" {

        grok {

            match => [ "message" , "(?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?<clientip>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server}?)(?:,
request: %{QS:request})?(?:, upstream: (?<upstream>\"%{URI}\"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \"%{URI:referrer}\")?"]

        }

    }

    # add geo-location info

    geoip {

        source => "clientip"

    }

}

output {

    elasticsearch {

        hosts => ["10.0.0.10"]

        index => "%{type}-%{+YYYY.MM.dd}"

    }

}

4 kibana分析效果图









参考资料
https://grokdebug.herokuapp.com/patterns# https://github.com/adventure-yunfei/ELK-for-nginx-log/blob/master/logstash.conf