spark读取Elasticsearch数据异常 java.lang.NullPointerException
Elasticsearch | 作者 mazedfishs | 发布于2017年09月22日 | 阅读数:8002
17/09/22 09:25:27 INFO JavaEsRDD: Reading from [client-behavior/ClickUnload]
Exception in thread "main" java.lang.NullPointerException
at org.elasticsearch.hadoop.rest.RestService.findPartitions(RestService.java:258)
at org.elasticsearch.spark.rdd.AbstractEsRDD.esPartitions$lzycompute(AbstractEsRDD.scala:59)
at org.elasticsearch.spark.rdd.AbstractEsRDD.esPartitions(AbstractEsRDD.scala:58)
at org.elasticsearch.spark.rdd.AbstractEsRDD.getPartitions(AbstractEsRDD.scala:26)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:217)
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:32)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:217)
at org.elasticsearch.spark.rdd.EsSpark$.doSaveToEs(EsSpark.scala:73)
at org.elasticsearch.spark.rdd.EsSpark$.saveToEs(EsSpark.scala:58)
at org.elasticsearch.spark.rdd.EsSpark$.saveToEs(EsSpark.scala:53)
at org.elasticsearch.spark.rdd.api.java.JavaEsSpark$.saveToEs(JavaEsSpark.scala:54)
at org.elasticsearch.spark.rdd.api.java.JavaEsSpark.saveToEs(JavaEsSpark.scala)
at com.cmb.spark.test.run(test.java:27)
at com.cmb.main.esread.main(esread.java:15)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144)
代码如下:
SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("TTT")
.set("es.nodes", "XXXXXXXX:9280")
.set("es.index.auto.create", "true")
.set("es.nodes.client.only", "false")
.set("es.nodes.wan.only","true")
.set("es.net.http.auth.user", "XXXXXXX")
.set("es.net.http.auth.pass", "XXXXXXXXX");
JavaSparkContext jsc = new JavaSparkContext(conf);
JavaRDD<Map<String, Object>> esRDD = JavaEsSpark.esRDD(jsc, "client-behavior/ClickUnload").values();
JavaEsSpark.saveToEs(esRDD, "esread/test");
Exception in thread "main" java.lang.NullPointerException
at org.elasticsearch.hadoop.rest.RestService.findPartitions(RestService.java:258)
at org.elasticsearch.spark.rdd.AbstractEsRDD.esPartitions$lzycompute(AbstractEsRDD.scala:59)
at org.elasticsearch.spark.rdd.AbstractEsRDD.esPartitions(AbstractEsRDD.scala:58)
at org.elasticsearch.spark.rdd.AbstractEsRDD.getPartitions(AbstractEsRDD.scala:26)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:217)
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:32)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:219)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:217)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:217)
at org.elasticsearch.spark.rdd.EsSpark$.doSaveToEs(EsSpark.scala:73)
at org.elasticsearch.spark.rdd.EsSpark$.saveToEs(EsSpark.scala:58)
at org.elasticsearch.spark.rdd.EsSpark$.saveToEs(EsSpark.scala:53)
at org.elasticsearch.spark.rdd.api.java.JavaEsSpark$.saveToEs(JavaEsSpark.scala:54)
at org.elasticsearch.spark.rdd.api.java.JavaEsSpark.saveToEs(JavaEsSpark.scala)
at com.cmb.spark.test.run(test.java:27)
at com.cmb.main.esread.main(esread.java:15)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144)
代码如下:
SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("TTT")
.set("es.nodes", "XXXXXXXX:9280")
.set("es.index.auto.create", "true")
.set("es.nodes.client.only", "false")
.set("es.nodes.wan.only","true")
.set("es.net.http.auth.user", "XXXXXXX")
.set("es.net.http.auth.pass", "XXXXXXXXX");
JavaSparkContext jsc = new JavaSparkContext(conf);
JavaRDD<Map<String, Object>> esRDD = JavaEsSpark.esRDD(jsc, "client-behavior/ClickUnload").values();
JavaEsSpark.saveToEs(esRDD, "esread/test");
1 个回复
mazedfishs
赞同来自: