即使是不成熟的尝试,也胜于胎死腹中的策略。

logstash消费topic是会将数据发送到全部的index中(有人知道怎么解决吗?)

匿名 | 发布于2019年11月12日 | 阅读数:231

架构: filebeat -->kafka集群--->logstash--->es集群---->kibana
filbeat收集多服务日志
logstash在消费topic到es时将只写到A topic中的数据写入到了 全部的索引中
 
logstash日志
[2019-11-12T14:38:25,938][WARN ][org.apache.kafka.common.utils.AppInfoParser] Error registering AppInfo mbean
javax.management.InstanceAlreadyExistsException: kafka.consumer:type=app-info,id=logstash-0
    at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:437) ~[?:1.8.0_181]
    at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerWithRepository(DefaultMBeanServerInterceptor.java:1898) ~[?:1.8.0_181]
    at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:966) ~[?:1.8.0_181]
    at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:900) ~[?:1.8.0_181]
    at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:324) ~[?:1.8.0_181]
    at com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:522) ~[?:1.8.0_181]
    at org.apache.kafka.common.utils.AppInfoParser.registerAppInfo(AppInfoParser.java:62) [kafka-clients-2.1.0.jar:?]
    at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:797) [kafka-clients-2.1.0.jar:?]
    at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:652) [kafka-clients-2.1.0.jar:?]
    at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:632) [kafka-clients-2.1.0.jar:?]
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) [?:1.8.0_181]
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) [?:1.8.0_181]
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) [?:1.8.0_181]
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423) [?:1.8.0_181]
    at org.jruby.javasupport.JavaConstructor.newInstanceDirect(JavaConstructor.java:247) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.java.invokers.ConstructorInvoker.call(ConstructorInvoker.java:63) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.java.invokers.ConstructorInvoker.call(ConstructorInvoker.java:141) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:80) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.java.proxies.ConcreteJavaProxy$InitializeMethod.call(ConcreteJavaProxy.java:48) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:80) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyClass.newInstance(RubyClass.java:915) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyClass$INVOKER$i$newInstance.call(RubyClass$INVOKER$i$newInstance.gen) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.java.proxies.ConcreteJavaProxy$NewMethod.call(ConcreteJavaProxy.java:109) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.targets.InvokeSite.invoke(InvokeSite.java:183) [jruby-complete-9.2.7.0.jar:?]
    at usr.local.logstash.vendor.bundle.jruby.$2_dot_5_dot_0.gems.logstash_minus_input_minus_kafka_minus_8_dot_3_dot_1.lib.logstash.inputs.kafka.RUBY$method$create_consumer$0(/usr/local/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-kafka-8.3.1/lib/logstash/inputs/kafka.rb:328) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:117) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:156) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:200) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:339) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:170) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.interpreter.InterpreterEngine.processCall(InterpreterEngine.java:317) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.interpreter.StartupInterpreterEngine.interpret(StartupInterpreterEngine.java:72) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.interpreter.Interpreter.INTERPRET_BLOCK(Interpreter.java:128) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.MixedModeIRBlockBody.commonYieldPath(MixedModeIRBlockBody.java:151) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.IRBlockBody.doYield(IRBlockBody.java:187) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.BlockBody.yield(BlockBody.java:116) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.Block.yield(Block.java:165) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyEnumerable$23.call(RubyEnumerable.java:849) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.CallBlock19.doYield(CallBlock19.java:98) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.BlockBody.yield(BlockBody.java:116) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.Block.yield(Block.java:165) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyFixnum.times(RubyFixnum.java:284) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyInteger$INVOKER$i$0$0$times.call(RubyInteger$INVOKER$i$0$0$times.gen) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.JavaMethod$JavaMethodZeroBlock.call(JavaMethod.java:555) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyClass.finvoke(RubyClass.java:505) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.Helpers.invoke(Helpers.java:433) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyBasicObject.callMethod(RubyBasicObject.java:393) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyEnumerator.each(RubyEnumerator.java:327) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyEnumerator$INVOKER$i$each.call(RubyEnumerator$INVOKER$i$each.gen) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyClass.finvoke(RubyClass.java:493) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.Helpers.invoke(Helpers.java:421) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyEnumerable.callEach19(RubyEnumerable.java:118) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyEnumerable.collectCommon(RubyEnumerable.java:841) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyEnumerable.map(RubyEnumerable.java:833) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyEnumerable$INVOKER$s$0$0$map.call(RubyEnumerable$INVOKER$s$0$0$map.gen) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.JavaMethod$JavaMethodZeroBlock.call(JavaMethod.java:555) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:296) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:82) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:89) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.instructions.CallBase.interpret(CallBase.java:537) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.interpreter.InterpreterEngine.processCall(InterpreterEngine.java:362) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.interpreter.StartupInterpreterEngine.interpret(StartupInterpreterEngine.java:72) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.interpreter.InterpreterEngine.interpret(InterpreterEngine.java:86) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.MixedModeIRMethod.INTERPRET_METHOD(MixedModeIRMethod.java:171) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:158) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:200) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.callsite.CachingCallSite.cacheAndCall(CachingCallSite.java:339) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:170) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.interpreter.InterpreterEngine.processCall(InterpreterEngine.java:317) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.interpreter.StartupInterpreterEngine.interpret(StartupInterpreterEngine.java:72) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.ir.interpreter.InterpreterEngine.interpret(InterpreterEngine.java:86) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.MixedModeIRMethod.INTERPRET_METHOD(MixedModeIRMethod.java:171) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:158) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:200) [jruby-complete-9.2.7.0.jar:?]
    at usr.local.logstash.logstash_minus_core.lib.logstash.pipeline.RUBY$block$start_input$1(/usr/local/logstash/logstash-core/lib/logstash/pipeline.rb:420) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.CompiledIRBlockBody.callDirect(CompiledIRBlockBody.java:136) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:77) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.runtime.Block.call(Block.java:124) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyProc.call(RubyProc.java:295) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyProc.call(RubyProc.java:274) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.RubyProc.call(RubyProc.java:270) [jruby-complete-9.2.7.0.jar:?]
    at org.jruby.internal.runtime.RubyRunnable.run(RubyRunnable.java:105) [jruby-complete-9.2.7.0.jar:?]
    at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]

 
已邀请:

要回复问题请先登录注册