服务器:centos6.7
安装logstash
sudo yum install logstash
启动: 其中scrapyd_conllect.conf为配置文件
nohup sudo /usr/share/logstash/bin/logstash -f scrapyd_collect.conf &
logstash收集python scrapy日志并发送给ES的config文件如下:
input {
file {
path=> [ "/var/log/spiders/*/*/*.log" ] #收集/var/log/spiders/文件夹下的日志文件
codec=>multiline {
pattern => "^%{TIMESTAMP_ISO8601} "
negate => true
what => "previous"
}
type=>"spider_log" #类型名称
# tags=>["XX.XX.XX.XX"]
}
}
##过滤
filter{
grok {
match => { "message" => "%{DATA:log_date} %{TIME:log_localtime} \[%{DATA:log_class}\] %{WORD:log_type}: %{GREEDYDATA}"}
}
# if [log_type] == "INFO"{ #可以根据抽取中的log_type进行过滤
# drop{}
# }
# if [log_type] == "DEBUG"{
# drop{}
# }
}
###output to es
output {
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "spider-log" # spider-log-%{+YYYY.MM.dd} 索引名称
#template_name => "mprobelog"
#document_id => "%{id}"
}
stdout { codec => json_lines }
}
来源:oschina
链接:https://my.oschina.net/u/4287583/blog/4334911