logstash 配置geoip 报The database provided is invalid or corrupted错
Logstash | 作者 zthua | 发布于2018年02月05日 | 阅读数:11380
logstash 配置问题:
input {
beats {
port => 5044
host => "192.168.3.231"
}
}
filter {
grok {
patterns_dir => ["/var/tmp/logstash-6.1.1/bin/patterns"]
match => { "message" => "%{NGINXACCESS}"}
}
geoip {
source => "message"
target => "geoip"
database => "/var/tmp/logstash-6.1.1/bin/GeoLiteCity.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
convert => [ "response","integer" ]
convert => [ "bytes","integer" ]
replace => { "type" => "nginx_access" }
remove_field => "message"
}
date {
match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
}
mutate {
remove_field => "timestamp"
}
}
output {
elasticsearch {
hosts => ['192.168.3.231:9200']
manage_template => false
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
}
}
patterns文件:
NGINXACCESS ["%{IPORHOST:clientip} - %{NOTSPACE:remote_user} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} \"%{IPV4:http_x_forwarded_for}\""]
启动logstash 总是报
Error registering plugin {:pipeline_id=>"main", :plugin=>"#<LogStash::FilterDelegator:0x30a296dd @metric_events_out=org.jruby.proxy.org.logstash.instrument.metrics.counter.LongCounter$Proxy2 - name: out value:0, @metric_events_in=org.jruby.proxy.org.logstash.instrument.metrics.counter.LongCounter$Proxy2 - name: in value:0, @logger=#<LogStash::Logging::Logger:0x78907102 @logger=#<Java::OrgApacheLoggingLog4jCore::Logger:0x1277c8ab>>, @metric_events_time=org.jruby.proxy.org.logstash.instrument.metrics.counter.LongCounter$Proxy2 - name: duration_in_millis value:0, @id=\"ac007340a9c44d148bb78517c4bab7d99b1780707a5feeaf3e1493e990bb5086\", @klass=LogStash::Filters::GeoIP, @metric_events=#<LogStash::Instrument::NamespacedMetric:0x6c2eea68 @metric=#<LogStash::Instrument::Metric:0x174d8e8e @collector=#<LogStash::Instrument::Collector:0x3bd33e5c @agent=nil, @metric_store=#<LogStash::Instrument::MetricStore:0x73f3037 @store=#<Concurrent::Map:0x00000000000fb4 entries=3 default_proc=nil>, @structured_lookup_mutex=#<Mutex:0x4598c098>, @fast_lookup=#<Concurrent::Map:0x00000000000fb8 entries=78 default_proc=nil>>>>, @namespace_name=[:stats, :pipelines, :main, :plugins, :filters, :ac007340a9c44d148bb78517c4bab7d99b1780707a5feeaf3e1493e990bb5086, :events]>, @filter=<LogStash::Filters::GeoIP source=>\"message\", target=>\"geoip\", database=>\"/var/tmp/logstash-6.1.1/bin/GeoLiteCity.dat\", add_field=>{\"[geoip][coordinates]\"=>[\"%{[geoip][longitude]}\", \"%{[geoip][latitude]}\"]}, id=>\"ac007340a9c44d148bb78517c4bab7d99b1780707a5feeaf3e1493e990bb5086\", enable_metric=>true, periodic_flush=>false, default_database_type=>\"City\", cache_size=>1000, tag_on_failure=>[\"_geoip_lookup_failure\"]>>", :error=>"The database provided is invalid or corrupted.", :thread=>"#<Thread:0x56500d85 run>"}
[2018-02-05T00:34:08,752][ERROR][logstash.pipeline ] Pipeline aborted due to error {:pipeline_id=>"main", :exception=>java.lang.IllegalArgumentException: The database provided is invalid or corrupted., :backtrace=>["org.logstash.filters.GeoIPFilter.<init>(org/logstash/filters/GeoIPFilter.java:72)", "java.lang.reflect.Constructor.newInstance(java/lang/reflect/Constructor.java:423)", "org.jruby.javasupport.JavaConstructor.newInstanceDirect(org/jruby/javasupport/JavaConstructor.java:246)", "org.jruby.RubyClass.newInstance(org/jruby/RubyClass.java:1022)", "org.jruby.RubyClass$INVOKER$i$newInstance.call(org/jruby/RubyClass$INVOKER$i$newInstance.gen)", "var.tmp.logstash_minus_6_dot_1_dot_1.vendor.bundle.jruby.$2_dot_3_dot_0.gems.logstash_minus_filter_minus_geoip_minus_5_dot_0_dot_2_minus_java.lib.logstash.filters.geoip.register(/var/tmp/logstash-6.1.1/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.2-java/lib/logstash/filters/geoip.rb:105)", "var.tmp.logstash_minus_6_dot_1_dot_1.vendor.bundle.jruby.$2_dot_3_dot_0.gems.logstash_minus_filter_minus_geoip_minus_5_dot_0_dot_2_minus_java.lib.logstash.filters.geoip.RUBY$method$register$0$__VARARGS__(var/tmp/logstash_minus_6_dot_1_dot_1/vendor/bundle/jruby/$2_dot_3_dot_0/gems/logstash_minus_filter_minus_geoip_minus_5_dot_0_dot_2_minus_java/lib/logstash/filters//var/tmp/logstash-6.1.1/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.2-java/lib/logstash/filters/geoip.rb)", "org.jruby.RubyClass.finvoke(org/jruby/RubyClass.java:522)", "org.jruby.RubyBasicObject.send19(org/jruby/RubyBasicObject.java:1684)", "org.jruby.RubyBasicObject$INVOKER$i$send19.call(org/jruby/RubyBasicObject$INVOKER$i$send19.gen)", "var.tmp.logstash_minus_6_dot_1_dot_1.vendor.jruby.lib.ruby.stdlib.forwardable.register(/var/tmp/logstash-6.1.1/vendor/jruby/lib/ruby/stdlib/forwardable.rb:189)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.register_plugin(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:343)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.block in register_plugins(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:354)", "org.jruby.RubyArray.each(org/jruby/RubyArray.java:1734)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.register_plugins(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:354)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.RUBY$method$register_plugins$0$__VARARGS__(var/tmp/logstash_minus_6_dot_1_dot_1/logstash_minus_core/lib/logstash//var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.maybe_setup_out_plugins(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:744)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.RUBY$method$maybe_setup_out_plugins$0$__VARARGS__(var/tmp/logstash_minus_6_dot_1_dot_1/logstash_minus_core/lib/logstash//var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.start_workers(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:364)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.RUBY$method$start_workers$0$__VARARGS__(var/tmp/logstash_minus_6_dot_1_dot_1/logstash_minus_core/lib/logstash//var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.run(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:288)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.RUBY$method$run$0$__VARARGS__(var/tmp/logstash_minus_6_dot_1_dot_1/logstash_minus_core/lib/logstash//var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.block in start(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:248)", "org.jruby.RubyProc.call(org/jruby/RubyProc.java:289)", "org.jruby.RubyProc.call(org/jruby/RubyProc.java:246)", "java.lang.Thread.run(java/lang/Thread.java:748)"], :thread=>"#<Thread:0x56500d85 run>"}
[2018-02-05T00:34:08,826][ERROR][logstash.agent ] Failed to execute action {:id=>:main, :action_type=>LogStash::ConvergeResult::FailedAction, :message=>"Could not execute action: LogStash::PipelineAction::Create/pipeline_id:main, action_result: false", :backtrace=>nil}
是什么原因呢
input {
beats {
port => 5044
host => "192.168.3.231"
}
}
filter {
grok {
patterns_dir => ["/var/tmp/logstash-6.1.1/bin/patterns"]
match => { "message" => "%{NGINXACCESS}"}
}
geoip {
source => "message"
target => "geoip"
database => "/var/tmp/logstash-6.1.1/bin/GeoLiteCity.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float" ]
convert => [ "response","integer" ]
convert => [ "bytes","integer" ]
replace => { "type" => "nginx_access" }
remove_field => "message"
}
date {
match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
}
mutate {
remove_field => "timestamp"
}
}
output {
elasticsearch {
hosts => ['192.168.3.231:9200']
manage_template => false
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
}
}
patterns文件:
NGINXACCESS ["%{IPORHOST:clientip} - %{NOTSPACE:remote_user} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} \"%{IPV4:http_x_forwarded_for}\""]
启动logstash 总是报
Error registering plugin {:pipeline_id=>"main", :plugin=>"#<LogStash::FilterDelegator:0x30a296dd @metric_events_out=org.jruby.proxy.org.logstash.instrument.metrics.counter.LongCounter$Proxy2 - name: out value:0, @metric_events_in=org.jruby.proxy.org.logstash.instrument.metrics.counter.LongCounter$Proxy2 - name: in value:0, @logger=#<LogStash::Logging::Logger:0x78907102 @logger=#<Java::OrgApacheLoggingLog4jCore::Logger:0x1277c8ab>>, @metric_events_time=org.jruby.proxy.org.logstash.instrument.metrics.counter.LongCounter$Proxy2 - name: duration_in_millis value:0, @id=\"ac007340a9c44d148bb78517c4bab7d99b1780707a5feeaf3e1493e990bb5086\", @klass=LogStash::Filters::GeoIP, @metric_events=#<LogStash::Instrument::NamespacedMetric:0x6c2eea68 @metric=#<LogStash::Instrument::Metric:0x174d8e8e @collector=#<LogStash::Instrument::Collector:0x3bd33e5c @agent=nil, @metric_store=#<LogStash::Instrument::MetricStore:0x73f3037 @store=#<Concurrent::Map:0x00000000000fb4 entries=3 default_proc=nil>, @structured_lookup_mutex=#<Mutex:0x4598c098>, @fast_lookup=#<Concurrent::Map:0x00000000000fb8 entries=78 default_proc=nil>>>>, @namespace_name=[:stats, :pipelines, :main, :plugins, :filters, :ac007340a9c44d148bb78517c4bab7d99b1780707a5feeaf3e1493e990bb5086, :events]>, @filter=<LogStash::Filters::GeoIP source=>\"message\", target=>\"geoip\", database=>\"/var/tmp/logstash-6.1.1/bin/GeoLiteCity.dat\", add_field=>{\"[geoip][coordinates]\"=>[\"%{[geoip][longitude]}\", \"%{[geoip][latitude]}\"]}, id=>\"ac007340a9c44d148bb78517c4bab7d99b1780707a5feeaf3e1493e990bb5086\", enable_metric=>true, periodic_flush=>false, default_database_type=>\"City\", cache_size=>1000, tag_on_failure=>[\"_geoip_lookup_failure\"]>>", :error=>"The database provided is invalid or corrupted.", :thread=>"#<Thread:0x56500d85 run>"}
[2018-02-05T00:34:08,752][ERROR][logstash.pipeline ] Pipeline aborted due to error {:pipeline_id=>"main", :exception=>java.lang.IllegalArgumentException: The database provided is invalid or corrupted., :backtrace=>["org.logstash.filters.GeoIPFilter.<init>(org/logstash/filters/GeoIPFilter.java:72)", "java.lang.reflect.Constructor.newInstance(java/lang/reflect/Constructor.java:423)", "org.jruby.javasupport.JavaConstructor.newInstanceDirect(org/jruby/javasupport/JavaConstructor.java:246)", "org.jruby.RubyClass.newInstance(org/jruby/RubyClass.java:1022)", "org.jruby.RubyClass$INVOKER$i$newInstance.call(org/jruby/RubyClass$INVOKER$i$newInstance.gen)", "var.tmp.logstash_minus_6_dot_1_dot_1.vendor.bundle.jruby.$2_dot_3_dot_0.gems.logstash_minus_filter_minus_geoip_minus_5_dot_0_dot_2_minus_java.lib.logstash.filters.geoip.register(/var/tmp/logstash-6.1.1/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.2-java/lib/logstash/filters/geoip.rb:105)", "var.tmp.logstash_minus_6_dot_1_dot_1.vendor.bundle.jruby.$2_dot_3_dot_0.gems.logstash_minus_filter_minus_geoip_minus_5_dot_0_dot_2_minus_java.lib.logstash.filters.geoip.RUBY$method$register$0$__VARARGS__(var/tmp/logstash_minus_6_dot_1_dot_1/vendor/bundle/jruby/$2_dot_3_dot_0/gems/logstash_minus_filter_minus_geoip_minus_5_dot_0_dot_2_minus_java/lib/logstash/filters//var/tmp/logstash-6.1.1/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.2-java/lib/logstash/filters/geoip.rb)", "org.jruby.RubyClass.finvoke(org/jruby/RubyClass.java:522)", "org.jruby.RubyBasicObject.send19(org/jruby/RubyBasicObject.java:1684)", "org.jruby.RubyBasicObject$INVOKER$i$send19.call(org/jruby/RubyBasicObject$INVOKER$i$send19.gen)", "var.tmp.logstash_minus_6_dot_1_dot_1.vendor.jruby.lib.ruby.stdlib.forwardable.register(/var/tmp/logstash-6.1.1/vendor/jruby/lib/ruby/stdlib/forwardable.rb:189)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.register_plugin(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:343)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.block in register_plugins(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:354)", "org.jruby.RubyArray.each(org/jruby/RubyArray.java:1734)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.register_plugins(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:354)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.RUBY$method$register_plugins$0$__VARARGS__(var/tmp/logstash_minus_6_dot_1_dot_1/logstash_minus_core/lib/logstash//var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.maybe_setup_out_plugins(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:744)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.RUBY$method$maybe_setup_out_plugins$0$__VARARGS__(var/tmp/logstash_minus_6_dot_1_dot_1/logstash_minus_core/lib/logstash//var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.start_workers(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:364)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.RUBY$method$start_workers$0$__VARARGS__(var/tmp/logstash_minus_6_dot_1_dot_1/logstash_minus_core/lib/logstash//var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.run(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:288)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.RUBY$method$run$0$__VARARGS__(var/tmp/logstash_minus_6_dot_1_dot_1/logstash_minus_core/lib/logstash//var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb)", "var.tmp.logstash_minus_6_dot_1_dot_1.logstash_minus_core.lib.logstash.pipeline.block in start(/var/tmp/logstash-6.1.1/logstash-core/lib/logstash/pipeline.rb:248)", "org.jruby.RubyProc.call(org/jruby/RubyProc.java:289)", "org.jruby.RubyProc.call(org/jruby/RubyProc.java:246)", "java.lang.Thread.run(java/lang/Thread.java:748)"], :thread=>"#<Thread:0x56500d85 run>"}
[2018-02-05T00:34:08,826][ERROR][logstash.agent ] Failed to execute action {:id=>:main, :action_type=>LogStash::ConvergeResult::FailedAction, :message=>"Could not execute action: LogStash::PipelineAction::Create/pipeline_id:main, action_result: false", :backtrace=>nil}
是什么原因呢
9 个回复
zthua
赞同来自:
zthua
赞同来自:
zthua
赞同来自:
evilbat
赞同来自:
sun_changlong
赞同来自:
zqc0512 - andy zhou
赞同来自:
zqc0512 - andy zhou
赞同来自:
sun_changlong
赞同来自:
hotyei2003
赞同来自:
if @database.nil?
@database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"GeoLite2-#{@default_database_type}.mmdb")).first
if @database.nil? || !File.exists?(@database)
raise "You must specify 'database => ...' in your geoip filter (I looked for '#{@database}')"
end
end
@logger.info("Using geoip database", :path => @database)
@geoipfilter = org.logstash.filters.GeoIPFilter.new(@source, @target, @fields, @database, @cache_size)
end #
代码报错应该是从这个地方开始的。我目前使用GeoLite2-City.mmdb没有出问题,但是IP解析没有GeoLiteCity.dat的数据全面,使用GeoLiteCity.dat就会报错。想问下关于这个错误,有能找到org/logstash/filters/GeoIPFilter.java:72源码的没