搜索结果正在快递途中

filebeat直接传数据到es，在es中加pipeline过滤某些字段，运行报错？？

Elasticsearch | 作者 Aruen | 发布于2019年02月23日 | 阅读数：5098
分享到：QQ空间新浪微博微信QQ好友印象笔记有道云笔记
 
org.elasticsearch.ElasticsearchException: java.lang.IllegalArgumentException: java.lang.IllegalArgumentException: field [date] not present as part of path [date]

	at org.elasticsearch.ingest.CompoundProcessor.newCompoundProcessorException(CompoundProcessor.java:156) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.CompoundProcessor.execute(CompoundProcessor.java:107) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.Pipeline.execute(Pipeline.java:58) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.PipelineExecutionService.innerExecute(PipelineExecutionService.java:166) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.PipelineExecutionService.access$000(PipelineExecutionService.java:41) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.PipelineExecutionService$2.doRun(PipelineExecutionService.java:88) [elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:527) [elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.1.1.jar:5.1.1]

	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_111]

	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_111]

	at java.lang.Thread.run(Thread.java:745) [?:1.8.0_111]

Caused by: java.lang.IllegalArgumentException: java.lang.IllegalArgumentException: field [date] not present as part of path [date]

	... 11 more

Caused by: java.lang.IllegalArgumentException: field [date] not present as part of path [date]

	at org.elasticsearch.ingest.IngestDocument.removeField(IngestDocument.java:293) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.IngestDocument.removeField(IngestDocument.java:270) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.common.RemoveProcessor.execute(RemoveProcessor.java:50) ~[?:?]

	at org.elasticsearch.ingest.CompoundProcessor.execute(CompoundProcessor.java:100) ~[elasticsearch-5.1.1.jar:5.1.1]

	... 9 more

[2019-02-23T13:11:46,836][DEBUG][o.e.a.i.IngestActionFilter] [32zHDET] failed to execute pipeline [timestamp-pipeline-id] for document [filebeat-2019.02.22/human/-1]

org.elasticsearch.ElasticsearchException: java.lang.IllegalArgumentException: java.lang.IllegalArgumentException: field [date] not present as part of path [date]

	at org.elasticsearch.ingest.CompoundProcessor.newCompoundProcessorException(CompoundProcessor.java:156) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.CompoundProcessor.execute(CompoundProcessor.java:107) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.Pipeline.execute(Pipeline.java:58) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.PipelineExecutionService.innerExecute(PipelineExecutionService.java:166) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.PipelineExecutionService.access$000(PipelineExecutionService.java:41) ~[elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.ingest.PipelineExecutionService$2.doRun(PipelineExecutionService.java:88) [elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:527) [elasticsearch-5.1.1.jar:5.1.1]

	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.1.1.jar:5.1.1]

	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_111]

	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_111]

	at java.lang.Thread.run(Thread.java:745) [?:1.8.0_111]
2019-02-23 添加评论

没有找到相关结果

    已邀请:

    rochy - rochy_he

    赞同来自:

    看报错信息是缺少字段，你可以设置 ignore_missing 为 true 来忽略报错信息

    ignore_missing
    When set to false, events that don’t contain any of the fields in match_pids will be discarded and an error will be generated. By default, this condition is ignored.
     
    2019-02-24 0 1
    分享

    Aruen - 算法工程师

    赞同来自: 

    output.elasticsearch:
    
#  index: "filebeat-%{+yyy.MM.dd}"
    
  hosts: ["ip1:9200"]
    
  #配置Pipeline过滤@timestamp
    
  pipeline: "timestamp-pipeline-id"
    
  indices:
    
    - index: "orange-%{[date]}"
    
      when.contains:
    
        type: "orange"
    
    - index: "apple-%{[date]}"
    
      when.contains:
    
        type: "apple"
    
    - index: "fruit-%{[date]}"
    
      when.contains:
    
        type: "fruit"
    
    - index: "vegetable-%{[date]}"
    
      when.contains:
    
        type: "vegetable"
    我filebeat输出到es是上面那样配置的，when.contains没有包含的字段时，
    会创建filebeat-%{+yyy.MM.dd}索引，可不可以设置成当没有满足条件的type时不创建索引？？？
     
    下面是默认创建的filebeat索引：
    {
    
   "_index": "filebeat-2019.02.23",
    
   "_type": "human",
    
   "_id": "-1",
    
   "_version": 3,
    
   "_score": 1,
    
   "_source": {
    
      "json": { },
    
      "message": ""
    
   }
    
}
    可不可以设置成当没有满足条件的type时不创建索引？？？
    2019-02-24 0 0
    分享

    Aruen - 算法工程师

    赞同来自: 

    我是直接在pipeline中加
    
"ignore_failure" : true,"ignore_missing" : true这样吗？？
    

    
curl -XPUT http://ip:9200/_ingest/pipelin ... ne-id -d'
    
{
    
  "processors" : [
    
	{
    
	  "remove" : {
    
		"field": "@timestamp"
    
	  }
    
	},
    
	{
    
	  "remove" : {
    
		"field": "type"
    
	  }
    
	},
    
	{
    
	  "remove" : {
    
		"field": "date"
    
	  }
    
	},
    
	"ignore_failure" : true,
    
	"ignore_missing" : true
    
  ]
    
}'
    2019-02-25 0 3
    分享
    为什么被折叠? 0 个回复被折叠

    要回复问题请先登录或注册

    Copyright © 2024 · CC BY-NC-SA 3.0

    本站服务器及带宽由 提供赞助