elk之nginx

elk之nginx:

ignore_older => 86400，不处理一天以前的文件。

zjtest7-frontend:/usr/local/logstash-2.3.4/config# cat logstash_agent.conf
input {
file {
type => "zj_nginx_access"
path => ["/rsyslog/data/nginx/zjzc/nginx_access0*_log.*"]
ignore_older => 87400
}
}
filter {
grok {
match => {
"message" => "%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?

<http_referer>\S+)\" \"(?<http_user_agent>\S+)\" \"(?<http_x_forwarded_for>\S+)\""
}
}
}
output {
redis {
host => "192.168.32.67"
data_type => "list"
key => "zj_nginx:redis"
port=>"6379"
password => "1234567"
}
}

启动logstash agent:

[elk@zjtest7-frontend sbin]$ cd /usr/local/logstash-2.3.4/bin/
[elk@zjtest7-frontend bin]$ ./logstash -f ../config/logstash_agent.conf

设置权限:

chown -R elk:elk /rsyslog

127.0.0.1:6379> keys *
1) "\xac\xed\x00\x05t\x00!message_left:20160630:18158464881"
2) "\xac\xed\x00\x05t\x00\x18contract_rebuild_qty:422"
3) "\xac\xed\x00\x05t\x00&oauth:c761feda1b6182c04864a54f8eee8344"
4) "\xac\xed\x00\x05t\x00Dapp_permission_cache:com.zjzc.common.vo.permission.AppPermissionBean"
5) "zj_nginx:redis"
6) "shiro_redis_session:42c9052e-9b60-4a1c-87a1-3aaa24a4369f"
7) "\xac\xed\x00\x05t\x003client_roles_cache:c761feda1b6182c04864a54f8eee8344"
8) "\xac\xed\x00\x05t\x00\x18contract_rebuild_qty:417"
9) "\xac\xed\x00\x05t\x00\x18contract_rebuild_qty:427"
10) "\xac\xed\x00\x05t\x00\x18contract_rebuild_qty:423"

127.0.0.1:6379> LLEN  "zj_nginx:redis"
(integer) 4232
127.0.0.1:6379>

zjtest7-frontend:/usr/local/logstash-2.3.4/config# cat logstash_indexer.conf
input {
redis {
host => "192.168.32.67"
data_type => "list"
key => "zj_nginx:redis"
type => "redis-input"
password => "1234567"
port =>"6379"
}
}
output {
elasticsearch {
hosts => "192.168.32.80:9200"
index => "logstash-zjzc-nginx-%{+YYYY.MM.dd}"
}
stdout {
codec => rubydebug
}
}