linux-安装logstash-6.6.2

环境

操作系统：ubuntu16.04

软件版本： filebeat-6.2.2-linux-x86_64

步骤

官网

https://www.elastic.co/cn/

下载

curl -L -O https://artifacts.elastic.co/downloads/logstash/logstash-6.6.2.tar.gz

注意版本需要与elasticsearch版本保持一样

解压

tar zvxf logstash-6.6.2.tar.gz

移动

mv logstash-6.6.2 /opt

进入

cd logstash-6.6.2/config

复制conf模板文件

cp logstash-sample.conf logstash.conf

修改默认配置

input {
  beats {
    port => 5044
  }
}
output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
}

启动

./bin/logstash -f ./config/logstash.conf

后台启动

./bin/logstash -f ./config/logstash.conf > /dev/null 2>&1 &

命令输出模式启动

./bin/logstash -e 'input{stdin{}}output{stdout{codec=>rubydebug}}'

问题

Exiting: 1 error: Error reading fileset mysql/error: Error reading manifest file: config file ("/opt/filebeat-6.2.2-linux-x86_64/module/mysql/error/manifest.yml") must be owned by the beat user (uid=0) or root

原因：这些检查的目的是防止未经授权的用户提供或修改Beat所运行的配置。配置文件的所有者必须root 是执行Beat进程的用户，或者是该用户。

解决方法：

官方说明：https://www.elastic.co/guide/en/beats/libbeat/5.3/config-file-permissions.html#config-file-permissions

To correct this problem you can use either chown root {beatname}.yml or chown 501 {beatname}.yml to change the owner of the configuration file.
Exiting: error loading config file: config file ("{beatname}.yml") can only be
writable by the owner but the permissions are "-rw-rw-r--" (to fix the
permissions use: 'chmod go-w /etc/{beatname}/{beatname}.yml')
To correct this problem, use chmod go-w /etc/{beatname}/{beatname}.yml to remove write privileges from anyone other than the owner.

通过chown root {beatname}.yml，将不同的yml文件授权给root用户，比如：filebeat-6.2.2-linux-x86_64/module/mysql下的*.yml,执行：chown root manifest.yml后重新./filebeat setup即可

参考文档

https://doc.yonyoucloud.com/doc/logstash-best-practice-cn/input/stdin.html

https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns

添加过滤配置

input {
  beats {
    port => 5044
  }
}
#过滤
filter {
    #nginx 日志过滤
    if "nginx-accesslog" in [tags] {
        grok {
                match => { "message" => "%{HTTPDATE:timestamp}\|%{IP:remote_addr}\|%{IPORHOST:http_host}\|(?:%{DATA:http_x_forwarded_for}|-)\|%{DATA:request_method}\|%{DATA:request_uri}\|%{DATA:server_protocol}\|%{NUMBER:status}\|(?:%{NUMBER:body_bytes_sent}|-)\|(?:%{DATA:http_referer}|-)\|%{DATA:http_user_agent}\|(?:%{DATA:request_time}|-)\|"}
        }
        mutate {
                convert => ["status","integer"]
                convert => ["body_bytes_sent","integer"]
                convert => ["request_time","float"]
        }
        geoip {
                source=>"remote_addr"
        }
        date {
                match => [ "timestamp","dd/MMM/YYYY:HH:mm:ss Z"]
        }
        useragent {
                source=>"http_user_agent"
        }
    }
    #logback日志过滤
    if "logback"  in [tags] {
        grok {         
#支持多个正则匹配，只到最佳 
   	      match => {"message" => "\[%{LOGLEVEL:level}\]\s\[%{DATA:thread}\]\s\[%{DATA:time}\]\s%{DATA:class}\s\[%{INT:line}\]\s\-\s(?<msg>.+)$"}
              #match => {"message" => "\[%{LOGLEVEL:level}\]\s\[%{DATA:thread}\]\s\[%{HTTPDATE:logdate}\]\s%{DATA:class}\s\[%{INT:line}\]\s\-\s(?<msg>.+)$"}            
              #match => {"message" => "\[%{LOGLEVEL:level}\]\s\[%{DATA:thread}\]\s\[(?<logdate>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME})\]\s%{DATA:class}\s\[%{INT:line}\]\s\-\s(?<msg>.+)$"} 
   		match => {"message" => "\-\s(?<msg>.+)$"}
                # 删除字段，防止重复存储相同内容
                remove_field => "message
                # 添加字段
                add_field => {
                     "device" => "logback"
                 }
        }
        date {  
                # 时间格式化
                match =>  [ "time", "YYYY-MM-DD  HH:mm:ss" ]
        }
    } 
    #log4j2过滤
    if "log4j2"  in [tags] {
        grok {
                match => {"message" => "%{DATA:timestamp} \[%{DATA:log_pid}\] %{LOGLEVEL:level}- %{DATA:log_message}" }
        }
        date {
                match =>  [ "timestamp", "YYYY-MM-DD  HH:mm:ss" ]
        }
    }
}
output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "java-logs-%{[@metadata][version]}-%{+yyyy.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
}

输入日志文件

input {
     file {
        path => "/usr/local/bin/contract/logs/*.log "
        start_position => "beginning"
        tags=> "logback"
        codec => json {
            charset => "UTF-8"
        }        
     }
     file {
        path => "/var/log/mysqld.log"
        start_position => "beginning"
        tags=> "sql"
        codec => json {
            charset => "UTF-8"
        }        
     }
}

输出到多个索引

output {
    # 通过自定义tags进行判断
    if "logback" in [tags] {
        elasticsearch {
            hosts => ["http://localhost:9200"]
            index => "java-logs-%{[@metadata][version]}-%{+yyyy.MM.dd}"
        }    
    }else {
        elasticsearch {
            hosts => ["http://localhost:9200"]
            index => "filebeat-%{[@metadata][version]}-%{+yyyy.MM.dd}"
        }          
    }       
}

过滤器配置

if [type] == "nginx" {
	grok {
	  match => { "message" => ["(?<RemoteIP>(\d*.\d*.\d*.\d*)) - %{DATA:[nginx][access][user_name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}\" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][referrer]}\" \"%{DATA:[nginx][access][agent]}\""] }
		add_field => {
		  "Device" => "Charles Desktop"
		}
		#删除字段
		remove_field => "message"
		remove_field => "beat.version"
		remove_field => "beat.name"
	}
}

来源：oschina

链接：https://my.oschina.net/u/437309/blog/4264038

标签

nginx

YML

Elastic

ElasticSearch

Beats

logback

grok