使用Logstash进行日志分析

LogStash主要用于数据收集和分析方面，配合Elasticsearch，Kibana用起来很方便，安装教程google出来很多。

推荐阅读

Elasticsearch 权威指南

精通 Elasticsearch

Kibana 中文指南

The Logstash Book

目的

输入常规的Nginx日志，过滤成所需的字段，存入Elasticsearch里面。

日志样式：

115.182.31.11 - - [02/Aug/2013:08:35:10 +0800] "GET /v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=%e6%96%97%e5%9c%b0%e4%b8%bb%e5%8d%95%e6%9c%ba%e7%89%88&uuid=861698005693444&client=1&operator=1&net=2&devicetype=1&adspacetype=1&category=2&ip=117.136.22.36&os_version=2.2.2&aw=320&ah=50×tamp=1375403699&density=1.5&pw=800&ph=480&Device=ZTE-U%2bV880&sign=1f6fd0992ca09e8525b0f7165a928a2a HTTP/1.1" 200 76 "-" "-" -
117.135.137.180 - - [02/Aug/2013:08:35:10 +0800] "GET /v2/get?Format=json&Key=47378200063c41fe90eff85f11ca4d2f&AppId=324&AppName=%25E5%258D%2595%25E6%259C%25BA%25E6%2596%2597%25E5%259C%25B0%25E4%25B8%25BB&Uuid=b51d63a91da5a4111e6cc1fb2c2538d5&Client=1&Operator=1&Net=2&DeviceType=1&AdSpaceType=1&Category=28&Ip=117.136.7.111&Os_version=4.0.4&Aw=320&Ah=50&TimeStamp=1375403708&Sign=9a00b63a04c165deea70dedd6b747697 HTTP/1.0" 200 776 "-" "-" -
115.182.31.11 - - [02/Aug/2013:08:35:10 +0800] "GET /v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=%e6%96%97%e5%9c%b0%e4%b8%bb%e5%8d%95%e6%9c%ba%e7%89%88&uuid=860173017274352&client=1&operator=2&net=3&devicetype=1&adspacetype=1&category=2&ip=120.7.195.5&os_version=2.3.5&aw=320&ah=50×tamp=1375403700&density=1.5&long=39&lat=0699733%2c116&pw=854&ph=480&Device=MI-ONE%2bPlus&sign=f65a4b2b2681ac489f65acf49e3d8ebd HTTP/1.1" 200 76 "-" "-" -
115.182.31.12 - - [02/Aug/2013:08:35:10 +0800] "GET /v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=%e6%96%97%e5%9c%b0%e4%b8%bb%e5%8d%95%e6%9c%ba%e7%89%88&uuid=863802017354171&client=1&operator=1&net=3&devicetype=1&adspacetype=1&category=2&ip=123.121.144.120&os_version=2.3.5&aw=320&ah=50×tamp=1375403698&density=1.5&long=40&lat=11183975%2c116&pw=854&ph=480&Device=MI-ONE%2bPlus&sign=0c74cf53a4b6adfe5e218f4fab920da3 HTTP/1.1" 200 76 "-" "-" -
115.182.31.8 - - [02/Aug/2013:08:35:10 +0800] "GET /v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=%e6%96%97%e5%9c%b0%e4%b8%bb%e5%8d%95%e6%9c%ba%e7%89%88&uuid=868247013598808&client=1&operator=4&net=2&devicetype=1&adspacetype=1&category=2&ip=117.136.20.88&os_version=2.3.5&aw=320&ah=50×tamp=1375403707&density=1.5&pw=800&ph=480&Device=Lenovo%2bA520GRAY&sign=43d5260eb2b89f5984b513067e074f5e HTTP/1.1" 200 67 "-" "-" -

经过Logstash抽取收集后，每段会输出形式：

{
"message" => "115.182.31.8 - - [02/Aug/2013:05:24:12 +0800] \"GET /v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=斗地主单机版&uuid=355696050506936&client=1&operator=1&net=3&devicetype=1&adspacetype=1&category=2&ip=113.228.122.247&os_version=4.1.1&aw=320&ah=50×tamp=1375392249&density=2&long=41&lat=917705,123&pw=1280&ph=720&Device=GT-N7100&sign=e9853bb1e8bd56874b647bc08e7ba576 HTTP/1.1\" 200 67 \"-\" \"-\" -",
"@version" => "1",
"@timestamp" => "2015-01-15T08:06:26.340Z",
"host" => "vovo",
"path" => "/home/vovo/access.log",
"client" => "1",
"ident" => "-",
"auth" => "-",
"timestamp" => "1375392249",
"verb" => "GET",
"request" => "/v2/get?key=0b0c1c5523aa40c3a5dcde4402947693&appid=153&appname=斗地主单机版&uuid=355696050506936&client=1&operator=1&net=3&devicetype=1&adspacetype=1&category=2&ip=113.228.122.247&os_version=4.1.1&aw=320&ah=50×tamp=1375392249&density=2&long=41&lat=917705,123&pw=1280&ph=720&Device=GT-N7100&sign=e9853bb1e8bd56874b647bc08e7ba576",
"http_version" => "1.1",
"response" => "200",
"bytes" => "67",
"key" => "0b0c1c5523aa40c3a5dcde4402947693",
"appid" => "153",
"appname" => "斗地主单机版",
"uuid" => "355696050506936",
"operator" => "1",
"net" => "3",
"devicetype" => "1",
"adspacetype" => "1",
"category" => "2",
"ip" => "113.228.122.247",
"os_version" => "4.1.1",
"aw" => "320",
"ah" => "50",
"density" => "2",
"long" => "41",
"lat" => "917705,123",
"pw" => "1280",
"ph" => "720",
"Device" => "GT-N7100",
"sign" => "e9853bb1e8bd56874b647bc08e7ba576"
}

为了方便理解和测试，我采用了Logstash样版配置文件进行设置。

sample.conf

当中包含实现urldecode和kv插件功能，需要运行 ./plugin install contrib 安装Logstash的默认插件。

input {
file {
path => "/home/vovo/access.log"  #指定日志目录或文件，也可以使用通配符*.log输入目录中的log文件。
start_position => "beginning"
}
}
filter {
grok {
match => ["message", "%{IPORHOST:client} (%{USER:ident}|-) (%{USER:auth}|-) \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version})?|-)\" %{NUMBER:response} %{NUMBER:bytes} \"(%{QS:referrer}|-)\" \"(%{QS:agent}|-)\""]
　　　　#匹配模式 message是每段读进来的日志，IP、HTTPDATE、WORD、NOTSPACE、NUMBER都是patterns/grok-patterns中定义好的正则格式名称，对照上面的日志进行编写,冒号，(?:%{USER:ident}|-)这种形式是条件判断，相当于程序里面的二目运算。如果有双引号""或者[]号，需要在前面加\进行转义。
}
kv {
source => "request"
field_split => "&?"
value_split => "="
}
　　#再单独将取得的URL、request字段取出来进行key-value值匹配，需要kv插件。提供字段分隔符"&?"，值键分隔符"="，则会自动将字段和值采集出来。
urldecode {
all_fields => true
}
　　#把所有字段进行urldecode（显示中文）
}

output {
#elasticsearch {
#    host => 'localhost'
#    protocol => "http"
#}
　　#把采集的数据输出到elasticsearch里面。
stdout {
codec => rubydebug
}
　　#输出到屏幕上
}