filebeat.yml配置和logstash.conf
2017-02-08 15:16
399 查看
#!/bin/bash
/home/haoren/data/logstash-5.0.2/bin/logstash -f afactivityserver.conf &
(1)
filebeat.prospectors:
- input_type: log
paths:
- /log/abcbillserver.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: billbijiesuan
fields_under_root: true
fields:
host: 192.168.10.7
processors:
- drop_fields:
#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
fields: ["input_type", "beat", "offset", "source"]
output.redis:
enabled: true
hosts: ["192.168.10.8"]
password: "A8841c09BAD52E63067C4DA"
port: 6379
datatype: list
key: "filebeat"
db: 0
output.file:
enabled: false
path: "/tmp/filebeat"
output.console:
enabled: false
(2)
filebeat.prospectors:
- input_type: log
paths:
- /log/cactivityserver.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: activityserver
fields_under_root: true
fields:
host: 192.168.10.13
- input_type: log
paths:
- /log/fvchannelserver.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: vchannelserver
fields_under_root: true
fields:
host: 192.168.10.13
processors:
- drop_fields:
#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
fields: ["input_type", "beat", "offset", "source"]
output.redis:
enabled: true
hosts: ["192.168.10.8"]
password: "A8841c09BAD52E63067C4DA"
port: 6379
datatype: list
key: "filebeat"
db: 0
output.file:
enabled: false
path: "/tmp/filebeat"
output.console:
enabled: false
logstash.conf 如下
input {
redis {
host => ["192.168.10.8"]
port => 6379
password => "A8841c09BAD52E63067C4DA"
data_type => "list"
key => "logstash"
codec => json {
charset => "UTF-8"
}
}
}
filter {
ruby {
code=>"event.set('daytag',event.timestamp.time.localtime.strftime('%Y.%m.%d'))"
}
}
output {
elasticsearch {
hosts => ["127.0.0.1:19200"]
index => "%{type}-%{daytag}"
#index => "%{type}-%{+yyyy.MM.dd}"
user => "logstashserver"
password => "A950C0FB2D833E42C1AC59210CD5CDF8"
}
}
cat /home/haoren/data/logstash-5.0.2/billserver.conf
input {
file {
path => "/log/abcbillserver.log"
codec => plain {
charset => "GBK"
}
#start_position => "beginning"
#sincedb_path => "/dev/null"
type => "billserver"
}
}
filter {
if ([message] =~ "人民币统计"){
mutate {replace => { "type" => "dubijiesuan" }}
grok {
#170208-10:00:28 Bill[40268] INFO: [人民币结算]时间(1486519228),用户ID(30581009),原来人民币(12567),现在人民币(16567),人民币操作(1),增加(4000),操作类型(19),操作详情(2),操作数量(0).描述:通用人民币操作
#170208-10:00:01 Bill[40268] INFO: [人民币结算]时间(1486519201),用户ID(22327945),原来人民币(2572),现在人民币(2532),人民币操作(2),扣除(40),操作类型(19),操作详情(15),操作数量(0).描述:通用人民币操作
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) Bill\[\d+\] INFO: \[人民币结算\]时间\(%{NUMBER:time:int}\),用户ID\(%{NUMBER:userid:int}\),原来人民币\(%{NUMBER:oldcoin:int}\),现在人民币\(%{NUMBER:currentcoin:int}\),人民币操作\(%{NUMBER:coinop:int}\),%{DATA:sop}\(%{NUMBER:coinnum:int}\),操作类型\(%{NUMBER:optype:int}\),操作详情\(%{NUMBER:opdetail:int}\),操作数量\(%{NUMBER:opnum:int}\).描述:%{DATA:sinfo}"]
}
}
else{
drop{}
}
date {
match => ["datetime", "yyMMdd-HH:mm:ss"]
#timezone => ["Asia/Hong_Kong"]
#remove_field => ["time"]
}
ruby {
code => "event.timestamp.time.localtime"
}
mutate {
#some pc no host
replace => { "host" => "192.168.10.7" }
}
}
output {
#stdout {
# codec => plain {
# charset => "UTF-8"
# #charset => "GBK"
# }
#}
#file {
# path => "/tmp/logstash.log"
# codec => json {
# charset => "UTF-8"
# }
#}
redis {
host => ["192.168.10.18"]
port => 6379
data_type => "list"
key => "logstash"
password => "A8841c09BAD52E63067C4DA"
codec => json {
charset => "UTF-8"
}
}
}
cat /home/haoren/data/logstash-5.0.2/activityserver.conf
input {
file {
path => "/log/aactivityserver.log"
codec => plain {
charset => "GBK"
}
#start_position => "beginning"
#sincedb_path => "/dev/null"
type => "aactivityserver"
}
}
filter {
if ([message] =~ "运统计"){
mutate {replace => { "type" => "activityescort" }}
grok {
#161201-13:12:28 ActivityServer[17701] INFO: [Escort.cpp:595] [统计]序号(53)用户(23619530)(攻)值(4360)暴击率(4)使用道具(57)本次花费(0)本总花费(0)车原始量(1706792)剩余量(1702432)总值(4360)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) ActivityServer\[\d+\].*?INFO: \[.*?\] \[运镖统计\]序号\(%{NUMBER:carid:int}\)用户\(%{NUMBER:userid:int}\)\(%{DATA:sop}\).*?\(%{NUMBER:itemvalue:int}\)暴击率\(%{NUMBER:baoji:int}\)使用道具\(%{NUMBER:itemid:int}\)本次花费\(%{NUMBER:coin:int}\)本总花费\(%{NUMBER:allusercoin:int}\)车原始量\(%{NUMBER:oldblood:int}\)剩余量\(%{NUMBER:blood:int}\)总值\(%{NUMBER:allvalue:int}\)"]
}
}
else if ([message] =~ "运通用奖励"){
mutate {replace => { "type" => "activityescort" }}
grok {
#161201-13:15:16 ActivityServer[17701] INFO: [运]序号(54), [运通用奖励] 用户(21772699)奖励方式(0)奖励类型(34)购物车ID(61)数量(16)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) ActivityServer\[\d+\] INFO: \[运\]序号\(%{NUMBER:carid:int}\), \[运通用奖励\] 用户\(%{NUMBER:userid:int}\)奖励方式\(%{NUMBER:method:int}\)奖励类型\(%{NUMBER:addtype:int}\)包裹ID\(%{NUMBER:itemid:int}\)数量\(%{NUMBER:num:int}\)"]
}
}
else{
drop{}
}
date {
match => ["datetime", "yyMMdd-HH:mm:ss"]
#timezone => ["Asia/Hong_Kong"]
#remove_field => ["time"]
}
ruby {
code => "event.timestamp.time.localtime"
}
mutate {
#some pc no host
replace => { "host" => "192.168.10.13" }
}
}
output {
#stdout {
# codec => plain {
# charset => "UTF-8"
# #charset => "GBK"
# }
#}
#file {
# path => "/tmp/logstash.log"
# codec => json {
# charset => "UTF-8"
# }
#}
redis {
host => ["192.168.10.8"]
port => 6379
data_type => "list"
key => "logstash"
password => "A8841c09BAD52E63067C4DA"
codec => json {
charset => "UTF-8"
}
}
}
cat /home/haoren/data/filebeat-5.0.2-linux-x86_64/filebeat.yml
filebeat.prospectors:
- input_type: log
paths:
- /log/billserver.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: billserver
fields_under_root: true
fields:
host: 192.168.10.7
processors:
- drop_fields:
#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
fields: ["input_type", "beat", "offset", "source"]
output.redis:
enabled: true
hosts: ["192.168.10.18"]
password: "A8841c09BAD52E63067C4DA"
port: 6379
datatype: list
key: "filebeat"
db: 0
output.file:
enabled: false
path: "/tmp/filebeat"
output.console:
enabled: false
/home/haoren/data/logstash-5.0.2/bin/logstash -f afactivityserver.conf &
(1)
filebeat.prospectors:
- input_type: log
paths:
- /log/abcbillserver.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: billbijiesuan
fields_under_root: true
fields:
host: 192.168.10.7
processors:
- drop_fields:
#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
fields: ["input_type", "beat", "offset", "source"]
output.redis:
enabled: true
hosts: ["192.168.10.8"]
password: "A8841c09BAD52E63067C4DA"
port: 6379
datatype: list
key: "filebeat"
db: 0
output.file:
enabled: false
path: "/tmp/filebeat"
output.console:
enabled: false
(2)
filebeat.prospectors:
- input_type: log
paths:
- /log/cactivityserver.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: activityserver
fields_under_root: true
fields:
host: 192.168.10.13
- input_type: log
paths:
- /log/fvchannelserver.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: vchannelserver
fields_under_root: true
fields:
host: 192.168.10.13
processors:
- drop_fields:
#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
fields: ["input_type", "beat", "offset", "source"]
output.redis:
enabled: true
hosts: ["192.168.10.8"]
password: "A8841c09BAD52E63067C4DA"
port: 6379
datatype: list
key: "filebeat"
db: 0
output.file:
enabled: false
path: "/tmp/filebeat"
output.console:
enabled: false
logstash.conf 如下
input {
redis {
host => ["192.168.10.8"]
port => 6379
password => "A8841c09BAD52E63067C4DA"
data_type => "list"
key => "logstash"
codec => json {
charset => "UTF-8"
}
}
}
filter {
ruby {
code=>"event.set('daytag',event.timestamp.time.localtime.strftime('%Y.%m.%d'))"
}
}
output {
elasticsearch {
hosts => ["127.0.0.1:19200"]
index => "%{type}-%{daytag}"
#index => "%{type}-%{+yyyy.MM.dd}"
user => "logstashserver"
password => "A950C0FB2D833E42C1AC59210CD5CDF8"
}
}
cat /home/haoren/data/logstash-5.0.2/billserver.conf
input {
file {
path => "/log/abcbillserver.log"
codec => plain {
charset => "GBK"
}
#start_position => "beginning"
#sincedb_path => "/dev/null"
type => "billserver"
}
}
filter {
if ([message] =~ "人民币统计"){
mutate {replace => { "type" => "dubijiesuan" }}
grok {
#170208-10:00:28 Bill[40268] INFO: [人民币结算]时间(1486519228),用户ID(30581009),原来人民币(12567),现在人民币(16567),人民币操作(1),增加(4000),操作类型(19),操作详情(2),操作数量(0).描述:通用人民币操作
#170208-10:00:01 Bill[40268] INFO: [人民币结算]时间(1486519201),用户ID(22327945),原来人民币(2572),现在人民币(2532),人民币操作(2),扣除(40),操作类型(19),操作详情(15),操作数量(0).描述:通用人民币操作
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) Bill\[\d+\] INFO: \[人民币结算\]时间\(%{NUMBER:time:int}\),用户ID\(%{NUMBER:userid:int}\),原来人民币\(%{NUMBER:oldcoin:int}\),现在人民币\(%{NUMBER:currentcoin:int}\),人民币操作\(%{NUMBER:coinop:int}\),%{DATA:sop}\(%{NUMBER:coinnum:int}\),操作类型\(%{NUMBER:optype:int}\),操作详情\(%{NUMBER:opdetail:int}\),操作数量\(%{NUMBER:opnum:int}\).描述:%{DATA:sinfo}"]
}
}
else{
drop{}
}
date {
match => ["datetime", "yyMMdd-HH:mm:ss"]
#timezone => ["Asia/Hong_Kong"]
#remove_field => ["time"]
}
ruby {
code => "event.timestamp.time.localtime"
}
mutate {
#some pc no host
replace => { "host" => "192.168.10.7" }
}
}
output {
#stdout {
# codec => plain {
# charset => "UTF-8"
# #charset => "GBK"
# }
#}
#file {
# path => "/tmp/logstash.log"
# codec => json {
# charset => "UTF-8"
# }
#}
redis {
host => ["192.168.10.18"]
port => 6379
data_type => "list"
key => "logstash"
password => "A8841c09BAD52E63067C4DA"
codec => json {
charset => "UTF-8"
}
}
}
cat /home/haoren/data/logstash-5.0.2/activityserver.conf
input {
file {
path => "/log/aactivityserver.log"
codec => plain {
charset => "GBK"
}
#start_position => "beginning"
#sincedb_path => "/dev/null"
type => "aactivityserver"
}
}
filter {
if ([message] =~ "运统计"){
mutate {replace => { "type" => "activityescort" }}
grok {
#161201-13:12:28 ActivityServer[17701] INFO: [Escort.cpp:595] [统计]序号(53)用户(23619530)(攻)值(4360)暴击率(4)使用道具(57)本次花费(0)本总花费(0)车原始量(1706792)剩余量(1702432)总值(4360)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) ActivityServer\[\d+\].*?INFO: \[.*?\] \[运镖统计\]序号\(%{NUMBER:carid:int}\)用户\(%{NUMBER:userid:int}\)\(%{DATA:sop}\).*?\(%{NUMBER:itemvalue:int}\)暴击率\(%{NUMBER:baoji:int}\)使用道具\(%{NUMBER:itemid:int}\)本次花费\(%{NUMBER:coin:int}\)本总花费\(%{NUMBER:allusercoin:int}\)车原始量\(%{NUMBER:oldblood:int}\)剩余量\(%{NUMBER:blood:int}\)总值\(%{NUMBER:allvalue:int}\)"]
}
}
else if ([message] =~ "运通用奖励"){
mutate {replace => { "type" => "activityescort" }}
grok {
#161201-13:15:16 ActivityServer[17701] INFO: [运]序号(54), [运通用奖励] 用户(21772699)奖励方式(0)奖励类型(34)购物车ID(61)数量(16)
match => [ "message", "(?(?>\d{6}-\d\d:\d\d:\d\d)) ActivityServer\[\d+\] INFO: \[运\]序号\(%{NUMBER:carid:int}\), \[运通用奖励\] 用户\(%{NUMBER:userid:int}\)奖励方式\(%{NUMBER:method:int}\)奖励类型\(%{NUMBER:addtype:int}\)包裹ID\(%{NUMBER:itemid:int}\)数量\(%{NUMBER:num:int}\)"]
}
}
else{
drop{}
}
date {
match => ["datetime", "yyMMdd-HH:mm:ss"]
#timezone => ["Asia/Hong_Kong"]
#remove_field => ["time"]
}
ruby {
code => "event.timestamp.time.localtime"
}
mutate {
#some pc no host
replace => { "host" => "192.168.10.13" }
}
}
output {
#stdout {
# codec => plain {
# charset => "UTF-8"
# #charset => "GBK"
# }
#}
#file {
# path => "/tmp/logstash.log"
# codec => json {
# charset => "UTF-8"
# }
#}
redis {
host => ["192.168.10.8"]
port => 6379
data_type => "list"
key => "logstash"
password => "A8841c09BAD52E63067C4DA"
codec => json {
charset => "UTF-8"
}
}
}
cat /home/haoren/data/filebeat-5.0.2-linux-x86_64/filebeat.yml
filebeat.prospectors:
- input_type: log
paths:
- /log/billserver.log
#- /home/haoren/data/filebeat-5.0.2-linux-x86_64/test.log
encoding: gbk
symlinks: true
include_lines: ['\[.*?统计\]','\[.*?结算\]']
document_type: billserver
fields_under_root: true
fields:
host: 192.168.10.7
processors:
- drop_fields:
#fields: ["beat.hostname", "beat.name", "beat.version", "input_type", "beat"]
fields: ["input_type", "beat", "offset", "source"]
output.redis:
enabled: true
hosts: ["192.168.10.18"]
password: "A8841c09BAD52E63067C4DA"
port: 6379
datatype: list
key: "filebeat"
db: 0
output.file:
enabled: false
path: "/tmp/filebeat"
output.console:
enabled: false
内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理 
相关文章推荐
- filebeat.yml配置和logstash.conf
- filebeat.yml（中文配置详解）
- 生产环境filebeat logstash配置模板
- 配置kibana和logstash、filebeat 日志统一收集
- elk日志分析filebeat配置（filebeat + logstash）
- ELK 架构之 Logstash 和 Filebeat 配置使用（采集过滤）
- filebeat+logstash配置搭建
- ELK 架构之 Elasticsearch、Kibana、Logstash 和 Filebeat 安装配置汇总（6.2.4 版本）
- ELK 架构之 Logstash 和 Filebeat 安装配置
- elk日志分析filebeat配置（filebeat + logstash）
- filebeat.yml（中文配置详解）
- Filebeat 日志收集器 logstash-forwarder 的替代品
- filebeat 配置文件详解
- filebeat-5.1.1配置详解
- lasticsearch 入门：Filebeat 安装及输出数据到 elasticsearch 或 logstash
- filebeat 配置文件详解
- Filebeat的高级配置-Filebeat部分
- ELK+Filebeat 安装配置入门
- logstash配合filebeat监控tomcat日志
- kibana ,logstash and filebeat