疲劳是最舒适的枕头,努力工作吧。

Filebeat input to Logstash 掛掉

Logstash | 作者 WalterX | 发布于2018年01月09日 | 阅读数:4260

logstash config
 
input {
  beats {
    port => 5044
    client_inactivity_timeout => 300
  }
}

filter{

  if "beats_input_codec_plain_applied" in [tags] {
        mutate {
            remove_tag => ["beats_input_codec_plain_applied"]
        }
    }
  if "_geoip_lookup_failure" in [tags] {
                drop { }
        }

  if "_grokparsefailure" in [tags] {
            drop { }
  }
  if [xclientip] == "-" {
      mutate{
                replace => { "xclientip" => "0.0.0.0" }
            }
        }
  if [type] in [ "AA", "BB", "CC" ] {

  grok{
      patterns_dir => ["/usr/local/logstash/patterns"]
      match => [ "message", "%{COMBINEDAPACHELOG2}" ]
    }
  geoip{
        source => "xclientip"
        target => "geoip"
        database => "/usr/local/logstash/GeoIP/GeoLite2-City.mmdb"
        add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
        add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
   }
        mutate{
        convert => [ "[geoip][coordinates]", "float", "bytes", "integer", "elapsedmillis", "integer" ]
        }
   date {
        match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" , "ISO8601" ]
        target => "@timestamp"

   }
   mutate {
     add_field => { "path" => "%{request}" }
   }
   mutate {
     gsub => [ "path", "\?.*", "" ]
}
   mutate {
     remove_field => [ "message", "source" ]
}

}
}

output {
if [type] in [ "AA", "BB", "CC" ] {
  elasticsearch {
    hosts =>[ "192.168.1.111:9200","192.168.1.110:9200" ]
    index => "%{type}-%{[@metadata][beat]}-%{+YYYY.MM.dd}"
#    flush_size => 50000
    timeout => "60"
    document_type => "%{[@metadata][type]}"
    template => "/usr/local/logstash/filebeat-index-template.json"
    template_overwrite => true
    user => 'elastic'
    password => 'changeme'
  }
}
}
 
 
 
Logstash  log 
 
[2018-01-09T17:34:54,987][FATAL][logstash.runner          ] An unexpected error occurred! {:er  ror=>#<InterruptedRegexpError: Regexp Interrupted>, :backtrace=>["org/jruby/RubyString.java:31  01:in `gsub'", "org/jruby/RubyString.java:3069:in `gsub'", "/usr/local/logstash/vendor/bundle/  jruby/1.9/gems/logstash-filter-mutate-3.2.0/lib/logstash/filters/mutate.rb:336:in `gsub_dynami  c_fields'", "/usr/local/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.2.0/lib  /logstash/filters/mutate.rb:327:in `gsub'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/l  ocal/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.2.0/lib/logstash/filters/m  utate.rb:309:in `gsub'", "/usr/local/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mut  ate-3.2.0/lib/logstash/filters/mutate.rb:223:in `filter'", "/usr/local/logstash/logstash-core/  lib/logstash/filters/base.rb:145:in `do_filter'", "/usr/local/logstash/logstash-core/lib/logst  ash/filters/base.rb:164:in `multi_filter'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/l  ocal/logstash/logstash-core/lib/logstash/filters/base.rb:161:in `multi_filter'", "/usr/local/l  ogstash/logstash-core/lib/logstash/filter_delegator.rb:46:in `multi_filter'", "(eval):583:in `  initialize'", "org/jruby/RubyArray.java:1613:in `each'", "(eval):575:in `initialize'", "org/jr  uby/RubyProc.java:281:in `call'", "(eval):338:in `filter_func'", "/usr/local/logstash/logstash  -core/lib/logstash/pipeline.rb:398:in `filter_batch'", "/usr/local/logstash/logstash-core/lib/  logstash/pipeline.rb:379:in `worker_loop'", "/usr/local/logstash/logstash-core/lib/logstash/pi  peline.rb:342:in `start_workers'"]}
 
已邀请:

要回复问题请先登录注册