logstash收集syslog日志

logstash收集syslog日志
注意：生产用syslog收集日志！！！

编写logstash配置文件

#首先我用rubydebug测试数据
[root@elk-node1 conf.d]# cat syslog.conf
input{
syslog{
type => "system-syslog"
host => "192.168.247.135"
port => "514"
}
}
output{
stdout{
codec => "rubydebug"
}
#检查语法
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/syslog.conf --configtest
Configuration OK
You have new mail in /var/spool/mail/root
[root@elk-node1 ~]# ss -lntp|grep 514
LISTEN     0      50      ::ffff:192.168.247.135:514                     :::*                   users:(("java",pid=9605,fd=14))
#修改rsyslog配置文件让其能访问
[root@elk-node1 ~]# vim /etc/rsyslog.conf
*.* @@192.168.247.135:514
[root@elk-node1 ~]# systemctl restart rsyslog
[root@elk-node1 ~]#
#运行测试
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/syslog.conf
Settings: Default filter workers: 1
Logstash startup completed
{
"message" => "Registered Authentication Agent for unix-process:9680:2638370 (system bus name :1.490 [/usr/bin/pkttyagent --notify-fd 5 --fallback], object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.UTF-8)\n",
"@version" => "1",
"@timestamp" => "2018-07-15T10:08:58.000Z",
"type" => "system-syslog",
"host" => "192.168.247.135",
"priority" => 85,
"timestamp" => "Jul 15 18:08:58",
"logsource" => "elk-node1",
"program" => "polkitd",
"pid" => "686",
"severity" => 5,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Notice"
}
#添加到elk-log.yml文件
[root@elk-node1 conf.d]# cat elk_log.conf
input {
file {
path => "/var/log/messages"
type => "system"
start_position => "beginning"
}
file {
path => "/var/log/elasticsearch/hejianlai.log"
type => "es-error"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
file {
path => "/var/log/nginx/access_json.log"
codec => json
start_position => "beginning"
type => "nginx-log"
}
syslog{
type => "system-syslog"
host => "192.168.247.135"
port => "514"
}
}
output {

if [type] == "system"{
elasticsearch {
hosts => ["192.168.247.135:9200"]
index => "systemlog-%{+YYYY.MM.dd}"
}
}

if [type] == "es-error"{
elasticsearch {
hosts => ["192.168.247.135:9200"]
index => "es-error-%{+YYYY.MM.dd}"
}
}
if [type] == "nginx-log"{
elasticsearch {
hosts => ["192.168.247.135:9200"]
index => "nginx-log-%{+YYYY.MM.dd}"
}
}
if [type] == "system-syslog"{
elasticsearch {
hosts => ["192.168.247.135:9200"]
index => "system-syslog-log-%{+YYYY.MM.dd}"
}
}
}

#检查语法
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf --configtestConfiguration OK
#后台运行
[root@elk-node1 conf.d]# ps aux|grep elk|awk '{print $2}'|xargs kill -9
kill: sending signal to 9780 failed: No such process
You have new mail in /var/spool/mail/root
[root@elk-node1 conf.d]# ps aux|grep elk|awk '{print $2}'
9785
[1]+  Killed                  /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf  (wd: ~)
(wd now: /etc/logstash/conf.d)
[root@elk-node1 conf.d]# ps aux|grep elk
root       9788  0.0  0.0 112704   972 pts/0    R+   18:18   0:00 grep --color=auto elk
[root@elk-node1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf &
[1] 9789
#手动添加日志
[root@elk-node1 conf.d]# logger "you hao"
[root@elk-node1 conf.d]# logger "hello world"
[root@elk-node1 conf.d]# logger "跟我一起学猫叫，一起喵喵喵"

Kibana设置

看hand插件上我们能看到system-syslog索引

Kibana上添加system-syslog索引

完美