CentOS7安装Logstash7.3从mysql导数据到ES

1. 下载

官网：https://www.elastic.co/downloads/logstash

7.3.0链接：https://pan.baidu.com/s/1SnIcXBNjWRBpnZJ1b3NUyA 提取码：lids

2. rz上传到/usr/local并解压

3. 测试

cd /usr/local/logstash-7.3.0
./bin/logstash -e ""

4. 安装logstash-input-jdbc

gem -v
yum install gem
./bin/logstash-plugin install logstash-input-jdbc

如果不成功可参考：https://blog.csdn.net/chenguihua5/article/details/90316484

5. 从mysql导入数据到es

下载mysql-connector-java驱动包（也可以从maven库找：repository\mysql\mysql-connector-java）

上传到/usr/local/lib（或者其他位置）

进入config目录设置配置文件

cd config/

配置mysql查询语句

vim jdbc.sql

例：

select * from flower_shop

配置模板

vim template.json

例：

{
	"template": "*",
	"version": 50001,
	"settings": {
		"index.refresh_interval": "5s"
	},
	"mappings": {
		"_default_": {
			"_all": {
				"enabled": true,
				"norms": false
			},
			"dynamic_templates": [
				{
					"message_field": {
						"path_match": "message",
						"match_mapping_type": "string",
						"mapping": {
							"type": "text",
							"norms": false
						}
					}
				},
				{
					"string_fields": {
						"match": "*",
						"match_mapping_type": "string",
						"mapping": {
							"type": "text",
							"norms": false,
							"analyzer": "ik_max_word",
							"fields": {
								"keyword": {
									"type": "keyword"
								}
							}
						}
					}
				}
			],
			"properties": {
				"@timestamp": {
					"type": "date",
					"include_in_all": false
				},
				"@version": {
					"type": "keyword",
					"include_in_all": false
				}
			}
		}
	}
}

配置启动执行文件

vim jdbc.conf

例：

input {
	stdin {
	}
	jdbc {
	  #换成自己的ip
	  jdbc_connection_string => "jdbc:mysql://127.0.0.1:3306/mydb?serverTimeZone=Asia/Shanghai"
	  jdbc_user => "root"
	  jdbc_password => "root"
	  jdbc_driver_library => "/usr/local/lib/mysql-connector-java-8.0.16.jar"
	  jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
	  jdbc_paging_enabled => "true"
	  jdbc_page_size => "50000"
	  statement_filepath => "/usr/local/logstash-7.3.0/config/jdbc.sql"
	  schedule => "* * * * *"
	  # type => "_doc"
	}
}

filter {
	json {
		source => "message"
		remove_field => ["message"]
	}
	ruby { 
		code => "event.set('timestamp', event.get('[@timestamp](https://my.oschina.net/u/1049939)').time.localtime + 8*60*60)" 
	}
	ruby {
		code => "event.set('[@timestamp](https://my.oschina.net/u/1049939)',event.get('timestamp'))"
	}
	mutate {
		remove_field => ["timestamp"]
	}
}

output {
	elasticsearch {
		hosts => "localhost:9200"
		index => "flower_shop"
		document_id => "%{id}"
		template_overwrite => true
		template => "/usr/local/logstash-7.3.0/config/template.json"
	}
	stdout {
		codec => json_lines
	}
}

启动测试

../bin/logstash -f jdbc.conf

来源：oschina

链接：https://my.oschina.net/joininjoy/blog/3168039

标签

logstash

CentOS

mysql

V-IM

flower