logstash导入oracle数据库数据到es,我需要自定义模板,希望有些字段不是test类型是keyword类型
这是我logstash.json文件模板的数据
{
"daa" : {
"template": "daa-*",
"settings": {
"index.number_of_shards": 3,
"number_of_replicas": 0
},
"mappings" : {
"logs" : {
"properties" : {
"@timestamp" : {
"type" : "date",
"format" : "dateOptionalTime",
"doc_values" : true
},
"@version" : {
"type" : "string",
"index" : "not_analyzed",
"doc_values" : true
},
"jh" : {
"type" : "keyword",
"index" : "not_analyzed"
}
}
}
}
}
}
这是我jdbc.config的数据
input {
jdbc {
jdbc_connection_string => "jdbc:oracle:thin:@192.168.0.254:1521:orcl"
jdbc_user => "ytsj"
jdbc_password => "ytsj"
jdbc_driver_library => "D:\loage\ojdbc6-11.2.0.3.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
connection_retry_attempts => "3"
jdbc_validate_connection => "true"
codec=>plain{charset=>"UTF-8"}
jdbc_page_size => 100000
jdbc_fetch_size => 10000
schedule => "* * * * *"
statement => "select * from DAA01"
type => "DAA01"
}
jdbc {
jdbc_connection_string => "jdbc:oracle:thin:@192.168.0.254:1521:orcl"
jdbc_user => "ytsj"
jdbc_password => "ytsj"
jdbc_driver_library => "D:\loage\ojdbc6-11.2.0.3.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
connection_retry_attempts => "3"
jdbc_validate_connection => "true"
codec=>plain{charset=>"UTF-8"}
jdbc_page_size => 100000
jdbc_fetch_size => 10000
schedule => "* * * * *"
statement => "select * from DAA01S"
type => "DAA01S"
}
jdbc {
jdbc_connection_string => "jdbc:oracle:thin:@192.168.0.254:1521:orcl"
jdbc_user => "ytsj"
jdbc_password => "ytsj"
jdbc_driver_library => "D:\loage\ojdbc6-11.2.0.3.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
connection_retry_attempts => "3"
jdbc_validate_connection => "true"
codec=>plain{charset=>"UTF-8"}
jdbc_page_size => 100000
jdbc_fetch_size => 10000
schedule => "* * * * *"
statement => "select * from DAA02"
type => "DAA02"
}
}
filter {
date {
match => ["time", "yyyy-MM-dd HH:mm:ss,SSS"]
}
json {
source => "message"
remove_field => ["message"]
}
}
output {
if[type]=="DAA01"{
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "daa-%{+YYYY.MM.dd}"
document_id => "%{jh}"
template => "logstash_jdbc_test/logstash.json"
template_name => "daa"
template_overwrite => "true"
}
}
if[type]=="DAA01S"{
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "daa-%{+YYYY.MM.dd}"
document_id => "%{jh}_%{ny}"
template => "logstash_jdbc_test/logstash.json"
template_name => "daa"
template_overwrite => "true"
}
}
if[type]=="DAA02"{
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "daa-%{+YYYY.MM.dd}"
document_id => "%{jh}"
template => "logstash_jdbc_test/logstash.json"
template_name => "daa"
template_overwrite => "true"
}
}
这是我上传报的错误
Failed to install template. {:message=>"Got response code '400' contacting Elasticsearch at URL 'http://127.0.0.1:9200/_template/daa'", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError", :backtrace=>["D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:80:in `perform_request'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:279:in `perform_request_to_url'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:266:in `perform_request'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:361:in `with_connection'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:265:in `perform_request'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:273:in `put'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client.rb:338:in `template_put'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client.rb:82:in `template_install'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/template_manager.rb:27:in `install'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/template_manager.rb:9:in `install_template'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/common.rb:58:in `install_template'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/common.rb:25:in `register'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:9:in `register'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/output_delegator.rb:43:in `register'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:290:in `register_plugin'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "org/jruby/RubyArray.java:1613:in `each'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:310:in `start_workers'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:235:in `run'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/agent.rb:408:in `start_pipeline'"]}
求解
这是我logstash.json文件模板的数据
{
"daa" : {
"template": "daa-*",
"settings": {
"index.number_of_shards": 3,
"number_of_replicas": 0
},
"mappings" : {
"logs" : {
"properties" : {
"@timestamp" : {
"type" : "date",
"format" : "dateOptionalTime",
"doc_values" : true
},
"@version" : {
"type" : "string",
"index" : "not_analyzed",
"doc_values" : true
},
"jh" : {
"type" : "keyword",
"index" : "not_analyzed"
}
}
}
}
}
}
这是我jdbc.config的数据
input {
jdbc {
jdbc_connection_string => "jdbc:oracle:thin:@192.168.0.254:1521:orcl"
jdbc_user => "ytsj"
jdbc_password => "ytsj"
jdbc_driver_library => "D:\loage\ojdbc6-11.2.0.3.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
connection_retry_attempts => "3"
jdbc_validate_connection => "true"
codec=>plain{charset=>"UTF-8"}
jdbc_page_size => 100000
jdbc_fetch_size => 10000
schedule => "* * * * *"
statement => "select * from DAA01"
type => "DAA01"
}
jdbc {
jdbc_connection_string => "jdbc:oracle:thin:@192.168.0.254:1521:orcl"
jdbc_user => "ytsj"
jdbc_password => "ytsj"
jdbc_driver_library => "D:\loage\ojdbc6-11.2.0.3.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
connection_retry_attempts => "3"
jdbc_validate_connection => "true"
codec=>plain{charset=>"UTF-8"}
jdbc_page_size => 100000
jdbc_fetch_size => 10000
schedule => "* * * * *"
statement => "select * from DAA01S"
type => "DAA01S"
}
jdbc {
jdbc_connection_string => "jdbc:oracle:thin:@192.168.0.254:1521:orcl"
jdbc_user => "ytsj"
jdbc_password => "ytsj"
jdbc_driver_library => "D:\loage\ojdbc6-11.2.0.3.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
connection_retry_attempts => "3"
jdbc_validate_connection => "true"
codec=>plain{charset=>"UTF-8"}
jdbc_page_size => 100000
jdbc_fetch_size => 10000
schedule => "* * * * *"
statement => "select * from DAA02"
type => "DAA02"
}
}
filter {
date {
match => ["time", "yyyy-MM-dd HH:mm:ss,SSS"]
}
json {
source => "message"
remove_field => ["message"]
}
}
output {
if[type]=="DAA01"{
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "daa-%{+YYYY.MM.dd}"
document_id => "%{jh}"
template => "logstash_jdbc_test/logstash.json"
template_name => "daa"
template_overwrite => "true"
}
}
if[type]=="DAA01S"{
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "daa-%{+YYYY.MM.dd}"
document_id => "%{jh}_%{ny}"
template => "logstash_jdbc_test/logstash.json"
template_name => "daa"
template_overwrite => "true"
}
}
if[type]=="DAA02"{
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "daa-%{+YYYY.MM.dd}"
document_id => "%{jh}"
template => "logstash_jdbc_test/logstash.json"
template_name => "daa"
template_overwrite => "true"
}
}
这是我上传报的错误
Failed to install template. {:message=>"Got response code '400' contacting Elasticsearch at URL 'http://127.0.0.1:9200/_template/daa'", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError", :backtrace=>["D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:80:in `perform_request'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:279:in `perform_request_to_url'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:266:in `perform_request'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:361:in `with_connection'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:265:in `perform_request'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:273:in `put'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client.rb:338:in `template_put'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client.rb:82:in `template_install'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/template_manager.rb:27:in `install'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/template_manager.rb:9:in `install_template'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/common.rb:58:in `install_template'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/common.rb:25:in `register'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:9:in `register'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/output_delegator.rb:43:in `register'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:290:in `register_plugin'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "org/jruby/RubyArray.java:1613:in `each'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:310:in `start_workers'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:235:in `run'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/agent.rb:408:in `start_pipeline'"]}
求解
2 个回复
zyy
赞同来自:
zqc0512 - andy zhou
赞同来自: