#StackBounty: #java #apache-spark #datastax-enterprise #datastax-startup How to start Spark Thrift Server on Datastax Enterprise (fails…

Bounty: 50

When I am trying to run datastax spark-sql-thriftserver, I am getting these errors:

dse spark-sql-thriftserver start --conf spark.cores.max=10 --conf spark.executor.memory=2g --hiveconf hive.server2.thrift.port=10001

Spark Command: /opt/jdk1.8.0_112/jre//bin/java -cp
/etc/dse/spark/:/usr/share/dse/spark/jars/*:/etc/dse/hadoop2-client/
-Djava.library.path=/usr/share/dse/hadoop2-client/lib/native:/usr/share/dse/cassandra/lib/sigar-bin:
-Dcassandra.logdir=/var/log/cassandra -XX:MaxHeapFreeRatio=50 -XX:MinHeapFreeRatio=20 -Dguice_include_stack_traces=OFF -Ddse.system_memory_in_mb=32174 -Dcassandra.config.loader=com.datastax.bdp.config.DseConfigurationLoader
-Dlogback.configurationFile=/etc/dse/spark/logback-spark.xml -Dcassandra.logdir=/var/log/cassandra -Ddse.client.configuration.impl=com.datastax.bdp.transport.client.HadoopBasedClientConfiguration
-Dderby.stream.error.method=com.datastax.bdp.derby.LogbackBridge.getLogger
-Xmx1024M org.apache.spark.deploy.SparkSubmit –conf spark.executor.memory=2g –conf spark.cores.max=10 –class
org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
spark-internal –hiveconf hive.server2.thrift.port=10001
======================================== WARN 2017-05-07 22:21:55 org.apache.spark.SparkContext: Use an existing SparkContext, some
configuration may not take effect. ERROR 2017-05-07 22:22:04
org.apache.spark.deploy.DseSparkSubmitBootstrapper: Failed to start
or submit Spark application java.lang.NoSuchMethodError:
org.apache.hive.service.cli.operation.LogDivertAppender.setWriter(Ljava/io/Writer;)V
at
org.apache.hive.service.cli.operation.LogDivertAppender.(LogDivertAppender.java:166)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.hive.service.cli.operation.OperationManager.initOperationLogCapture(OperationManager.java:85)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.hive.service.cli.operation.OperationManager.init(OperationManager.java:63)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.ReflectedCompositeService$$anonfun$initCompositeService$1.apply(SparkSQLCLIService.scala:79)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.ReflectedCompositeService$$anonfun$initCompositeService$1.apply(SparkSQLCLIService.scala:79)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
scala.collection.Iterator$class.foreach(Iterator.scala:893)
~[scala-library-2.11.8.jar:na] at
scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
~[scala-library-2.11.8.jar:na] at
scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
~[scala-library-2.11.8.jar:na] at
scala.collection.AbstractIterable.foreach(Iterable.scala:54)
~[scala-library-2.11.8.jar:na] at
org.apache.spark.sql.hive.thriftserver.ReflectedCompositeService$class.initCompositeService(SparkSQLCLIService.scala:79)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.SparkSQLSessionManager.initCompositeService(SparkSQLSessionManager.scala:36)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.SparkSQLSessionManager.init(SparkSQLSessionManager.scala:58)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.ReflectedCompositeService$$anonfun$initCompositeService$1.apply(SparkSQLCLIService.scala:79)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.ReflectedCompositeService$$anonfun$initCompositeService$1.apply(SparkSQLCLIService.scala:79)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
scala.collection.Iterator$class.foreach(Iterator.scala:893)
~[scala-library-2.11.8.jar:na] at
scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
~[scala-library-2.11.8.jar:na] at
scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
~[scala-library-2.11.8.jar:na] at
scala.collection.AbstractIterable.foreach(Iterable.scala:54)
~[scala-library-2.11.8.jar:na] at
org.apache.spark.sql.hive.thriftserver.ReflectedCompositeService$class.initCompositeService(SparkSQLCLIService.scala:79)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.SparkSQLCLIService.initCompositeService(SparkSQLCLIService.scala:39)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.SparkSQLCLIService.init(SparkSQLCLIService.scala:62)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.ReflectedCompositeService$$anonfun$initCompositeService$1.apply(SparkSQLCLIService.scala:79)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.ReflectedCompositeService$$anonfun$initCompositeService$1.apply(SparkSQLCLIService.scala:79)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
scala.collection.Iterator$class.foreach(Iterator.scala:893)
~[scala-library-2.11.8.jar:na] at
scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
~[scala-library-2.11.8.jar:na] at
scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
~[scala-library-2.11.8.jar:na] at
scala.collection.AbstractIterable.foreach(Iterable.scala:54)
~[scala-library-2.11.8.jar:na] at
org.apache.spark.sql.hive.thriftserver.ReflectedCompositeService$class.initCompositeService(SparkSQLCLIService.scala:79)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.HiveThriftServer2.initCompositeService(HiveThriftServer2.scala:272)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.HiveThriftServer2.init(HiveThriftServer2.scala:292)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.HiveThriftServer2$.main(HiveThriftServer2.scala:94)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.sql.hive.thriftserver.HiveThriftServer2.main(HiveThriftServer2.scala)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[na:1.8.0_112] at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[na:1.8.0_112] at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[na:1.8.0_112] at java.lang.reflect.Method.invoke(Method.java:498)
~[na:1.8.0_112] at
org.apache.spark.deploy.DseSparkSubmit$.org$apache$spark$deploy$DseSparkSubmit$$runMain(DseSparkSubmit.scala:730)
~[dse-spark-5.1.0.jar:5.1.0] at
org.apache.spark.deploy.DseSparkSubmit$.doRunMain$1(DseSparkSubmit.scala:175)
~[dse-spark-5.1.0.jar:5.1.0] at
org.apache.spark.deploy.DseSparkSubmit$.submit(DseSparkSubmit.scala:200)
~[dse-spark-5.1.0.jar:5.1.0] at
org.apache.spark.deploy.DseSparkSubmit$.main(DseSparkSubmit.scala:109)
~[dse-spark-5.1.0.jar:5.1.0] at
org.apache.spark.deploy.DseSparkSubmitBootstrapper$.main(DseSparkSubmitBootstrapper.scala:74)
~[dse-spark-5.1.0.jar:5.1.0] at
org.apache.spark.deploy.DseSparkSubmitBootstrapper.main(DseSparkSubmitBootstrapper.scala)
[dse-spark-5.1.0.jar:5.1.0] ERROR 2017-05-07 22:22:15
org.apache.spark.util.Utils: Uncaught exception in thread Thread-0
java.lang.NullPointerException: null at
org.apache.spark.sql.hive.thriftserver.HiveThriftServer2$$anonfun$main$1.apply$mcV$sp(HiveThriftServer2.scala:85)
~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:215)
~[spark-core_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:187)
~[spark-core_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:187)
~[spark-core_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:187)
~[spark-core_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1953)
~[spark-core_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:187)
[spark-core_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:187)
[spark-core_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:187)
[spark-core_2.11-2.0.2.6.jar:2.0.2.6] at
scala.util.Try$.apply(Try.scala:192) [scala-library-2.11.8.jar:na] at
org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:187)
[spark-core_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:177)
[spark-core_2.11-2.0.2.6.jar:2.0.2.6] at
org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54)
[hadoop-common-2.7.1.3.jar:na]

Relevant part of the log:
ERROR 2017-05-07 22:22:04
org.apache.spark.deploy.DseSparkSubmitBootstrapper: Failed to start or submit Spark application java.lang.NoSuchMethodError:
org.apache.hive.service.cli.operation.LogDivertAppender.setWriter(Ljava/io/Writer;)V at
org.apache.hive.service.cli.operation.LogDivertAppender.(LogDivertAppender.java:166) ~[spark-hive-thriftserver_2.11-2.0.2.6.jar:2.0.2.6]

I have opscenter 6.1 and dse 5.1

====
Update 1:

files in /usr/share/dse/hadoop2-client/lib/native

libhadoop.a libhadoop.so libhadoop.so.1.0.0 libhadooppipes.a
libhadooputils.a libhdfs.a libhdfs.so libhdfs.so.0.0.0

files in /usr/share/dse/spark/jars

empty

files in /usr/share/dse/spark/lib

JavaEWAH-0.3.2.jar commons-dbcp-1.4.jar
httpclient-4.5.2.jar jersey-server-2.22.2.jar
netty-3.9.8.Final.jar reflectasm-1.10.1.jar
spark-sketch_2.11-2.0.2.6.jar RoaringBitmap-0.5.11.jar
commons-io-2.5.jar httpcore-4.4.4.jar
jline-2.14.2.jar noggit-0.6.jar
scala-compiler-2.11.8.jar
spark-sql_2.11-2.0.2.6.jar antlr4-runtime-4.5.3.jar
commons-lang3-3.4.jar httpmime-4.4.1.jar
joda-convert-1.2.jar objenesis-2.1.jar
scala-xml_2.11-1.0.4.jar
spark-streaming_2.11-2.0.2.6.jar aopalliance-repackaged-2.4.0-b34.jar
commons-math3-3.4.1.jar ivy-2.4.0.jar
joda-time-2.9.3.jar opencsv-2.3.jar
scalap-2.11.8.jar
spark-tags_2.11-2.0.2.6.jar arpack_combined_all-0.1.jar
compress-lzf-1.0.3.jar jackson-annotations-2.5.3.jar
jodd-core-3.5.2.jar oro-2.0.8.jar
scalatest_2.11-2.2.6.jar
spark-unsafe_2.11-2.0.2.6.jar avro-1.7.7.jar
core-1.1.2.jar jackson-core-asl-1.9.13.jar
jpam-1.1.jar osgi-resource-locator-1.0.1.jar
snappy-0.2.jar
spire-macros_2.11-0.7.4.jar avro-ipc-1.7.7.jar
datanucleus-api-jdo-3.2.6.jar jackson-mapper-asl-1.9.13.jar
json4s-ast_2.11-3.2.11.jar paranamer-2.8.jar
solr-solrj-6.0.1.0.1596.jar
spire_2.11-0.7.4.jar avro-mapred-1.7.7-hadoop2.jar
datanucleus-core-3.2.10.jar jackson-module-scala_2.11-2.5.3.jar
json4s-core_2.11-3.2.11.jar parquet-column-1.7.0.jar
spark-cassandra-connector-unshaded_2.11-2.0.1.jar stax-api-1.0.1.jar
bonecp-0.8.0.RELEASE.jar datanucleus-rdbms-3.2.9.jar
javax.annotation-api-1.2.jar
json4s-jackson_2.11-3.2.11.jar parquet-common-1.7.0.jar
spark-catalyst_2.11-2.0.2.6.jar stax2-api-3.1.4.jar
breeze-macros_2.11-0.11.2.jar derby-10.10.2.0.jar
javax.inject-2.4.0-b34.jar jsr166e-1.1.0.jar
parquet-encoding-1.7.0.jar spark-core_2.11-2.0.2.6.jar
stream-2.7.0.jar breeze_2.11-0.11.2.jar
eigenbase-properties-1.1.5.jar javax.ws.rs-api-2.0.1.jar
jta-1.1.jar parquet-format-2.3.0-incubating.jar
spark-graphx_2.11-2.0.2.6.jar super-csv-2.2.0.jar
calcite-avatica-1.2.0-incubating.jar hive-beeline-1.2.1.spark2.jar
javolution-5.5.1.jar jtransforms-2.4.0.jar
parquet-generator-1.7.0.jar
spark-hive-thriftserver_2.11-2.0.2.6.jar
univocity-parsers-2.1.1.jar calcite-core-1.2.0-incubating.jar
hive-cli-1.2.1.spark2.jar jdo-api-3.0.1.jar
jul-to-slf4j-1.7.13.jar parquet-hadoop-1.7.0.jar
spark-hive_2.11-2.0.2.6.jar unused-1.0.0.jar
calcite-linq4j-1.2.0-incubating.jar hive-exec-1.2.1.spark2.jar
jersey-client-2.22.2.jar kryo-3.0.3.jar
parquet-hadoop-bundle-1.6.0.jar spark-launcher_2.11-2.0.2.6.jar
velocity-1.7.jar cassandra-driver-mapping-3.1.4.jar
hive-jdbc-1.2.1.spark2.jar jersey-common-2.22.2.jar
libfb303-0.9.3.jar parquet-jackson-1.7.0.jar
spark-mllib-local_2.11-2.0.2.6.jar
woodstox-core-asl-4.4.1.jar chill-java-0.8.0.jar
hive-metastore-1.2.1.spark2.jar jersey-container-servlet-2.22.2.jar
mail-1.4.7.jar pmml-model-1.2.15.jar
spark-mllib_2.11-2.0.2.6.jar
xbean-asm5-shaded-4.4.jar chill_2.11-0.8.0.jar
hk2-api-2.4.0-b34.jar
jersey-container-servlet-core-2.22.2.jar
mesos-0.21.1-shaded-protobuf.jar pmml-schema-1.2.15.jar
spark-network-common_2.11-2.0.2.6.jar commons-beanutils-1.9.3.jar
hk2-locator-2.4.0-b34.jar jersey-guava-2.22.2.jar
metrics-json-3.1.2.jar py4j-0.10.1.jar
spark-network-shuffle_2.11-2.0.2.6.jar commons-codec-1.10.jar
hk2-utils-2.4.0-b34.jar jersey-media-jaxb-2.22.2.jar
minlog-1.3.0.jar pyrolite-4.13.jar
spark-repl_2.11-2.0.2.6.jar


Get this bounty!!!