Jdbc向es集群传输数据并解析json格式
匿名 | 发布于2018年09月14日 | 阅读数:3093
mysql里面的数据是这样的:
id phone data
12 12453452 {"cc":"aex","c":" qkp ","s":"owj","pt":"124sqa"}
我的配置文件是这样的:
input {
jdbc {
jdbc_driver_library => "C:/ES/mysql-connector-java-5.1.46-bin.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://localhost:3306/aa?autoReconnect=true&useSSL=false"
jdbc_user => ""
jdbc_password => ""
#schedule => "* * * * *"
jdbc_default_timezone => "Asia/Shanghai"
statement => "SELECT * FROM a;"
}
}
filter {
json {
source => "data"
add_field => {
"data_cc" => "%{cc}"
"data_c" => "%{c}"
"data_s" => "%{s}"
"data_pt" => "%{pt}"
}
add_field => {"source" => ""}
remove_field => ["_id" , "@version", "@timestamp", "path", "host","tags","message","data","cc","c","s","pt"]
}
}
output {
elasticsearch {
index => ""
document_type => ""
#document_id => "%{id}"
user => ""
password => ""
hosts => [""]
}
stdout {
codec => rubydebug
}
}
输出的是这样:
{
"source" => "",
"data_s" => "owj",
"tags" => ,
"data_cc" => "aex",
"phone" => "12453452 ",
"id" => 12,
"data_pt" => "124sqa",
"data_c" => "qkp "
}
我想要这样的结果:
{
"source" => "",
"tags" => ,
"phone" => "",
"id" => 12,
data => {
"data_pt" => "124sqa",
"data_c" => "qkp ",
"data_s" => "owj",
"data_cc" => "aex"
}
}
配置文件应该怎么写.
id phone data
12 12453452 {"cc":"aex","c":" qkp ","s":"owj","pt":"124sqa"}
我的配置文件是这样的:
input {
jdbc {
jdbc_driver_library => "C:/ES/mysql-connector-java-5.1.46-bin.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://localhost:3306/aa?autoReconnect=true&useSSL=false"
jdbc_user => ""
jdbc_password => ""
#schedule => "* * * * *"
jdbc_default_timezone => "Asia/Shanghai"
statement => "SELECT * FROM a;"
}
}
filter {
json {
source => "data"
add_field => {
"data_cc" => "%{cc}"
"data_c" => "%{c}"
"data_s" => "%{s}"
"data_pt" => "%{pt}"
}
add_field => {"source" => ""}
remove_field => ["_id" , "@version", "@timestamp", "path", "host","tags","message","data","cc","c","s","pt"]
}
}
output {
elasticsearch {
index => ""
document_type => ""
#document_id => "%{id}"
user => ""
password => ""
hosts => [""]
}
stdout {
codec => rubydebug
}
}
输出的是这样:
{
"source" => "",
"data_s" => "owj",
"tags" => ,
"data_cc" => "aex",
"phone" => "12453452 ",
"id" => 12,
"data_pt" => "124sqa",
"data_c" => "qkp "
}
我想要这样的结果:
{
"source" => "",
"tags" => ,
"phone" => "",
"id" => 12,
data => {
"data_pt" => "124sqa",
"data_c" => "qkp ",
"data_s" => "owj",
"data_cc" => "aex"
}
}
配置文件应该怎么写.
2 个回复
zyy
赞同来自: JonRambo
"data_cc" => "%{cc}"
"data_c" => "%{c}"
"data_s" => "%{s}"
"data_pt" => "%{pt}"
}
写成下面这样就可以了
add_field => {
"[data][data_cc]" => "%{cc}"
"[data][data_c]" => "%{c}"
"[data][data_s]" => "%{s}"
"[data][data_pt]" => "%{pt}"
}
JonRambo
赞同来自: