NoClassDefFoundError: org/apache/spark/Logging

从 tensorframes/core.py 开始，重点检查 _java_api() 和 extra_schema_info 调用，并使用报告中的命令 pyspark --packages databricks:tensorframes:0.2.3-s_2.10。比较该 artifact 与 Spark 1.6、Spark 2.0 以及自定义 branch 的兼容性；完成的标准是明确确定受支持的 setup 或所需的兼容性更改，并且 NoClassDefFoundError 已解决或记录在案。

I am trying to run sample code with $pyspark --packages databricks:tensorframes:0.2.3-s_2.10
I am getting this error (below attached) with spark-1.6, spark-2.0 and custom build on spark with branch-2.0.
I tried to find org.apache.spark.logging jar, which I am unable to find anywhere.
I am using Mac OS ElCapitain.

Please help.

Py4JJavaError Traceback (most recent call last)
in ()
8 # The TensorFlow placeholder that corresponds to column 'x'.
9 # The shape of the placeholder is automatically inferred from the DataFrame.
---> 10 x = tfs.block(df, "x")
11 # The output that adds 3 to x
12 z = tf.add(x, 3, name='z')

/private/var/folders/_f/fzxfs9dj33j0381wvy5bdsdm0000gn/T/spark-1831a94f-7cdd-4691-a32a-4cbc3d9bc3a5/userFiles-66cd46aa-619d-41cf-806d-3dc6e509e13d/databricks_tensorframes-0.2.3-s_2.10.jar/tensorframes/core.py in block(df, col_name, tf_name)
313 :return: a TensorFlow placeholder.
314 """
--> 315 return _auto_placeholder(df, col_name, tf_name, block = True)
316
317 def row(df, col_name, tf_name = None):

/private/var/folders/_f/fzxfs9dj33j0381wvy5bdsdm0000gn/T/spark-1831a94f-7cdd-4691-a32a-4cbc3d9bc3a5/userFiles-66cd46aa-619d-41cf-806d-3dc6e509e13d/databricks_tensorframes-0.2.3-s_2.10.jar/tensorframes/core.py in _auto_placeholder(df, col_name, tf_name, block)
331
332 def _auto_placeholder(df, col_name, tf_name, block):
--> 333 info = _java_api().extra_schema_info(df._jdf)
334 col_shape = [x.shape() for x in info if x.fieldName() == col_name]
335 if len(col_shape) == 0:

/private/var/folders/_f/fzxfs9dj33j0381wvy5bdsdm0000gn/T/spark-1831a94f-7cdd-4691-a32a-4cbc3d9bc3a5/userFiles-66cd46aa-619d-41cf-806d-3dc6e509e13d/databricks_tensorframes-0.2.3-s_2.10.jar/tensorframes/core.py in _java_api()
28 # You cannot simply call the creation of the the class on the _jvm due to classloader issues
29 # with Py4J.
---> 30 return _jvm.Thread.currentThread().getContextClassLoader().loadClass(javaClassName)
31 .newInstance()
32

/usr/local/spark/python/lib/py4j-0.10.1-src.zip/py4j/java_gateway.py in call(self, *args)
931 answer = self.gateway_client.send_command(command)
932 return_value = get_return_value(
--> 933 answer, self.gateway_client, self.target_id, self.name)
934
935 for temp_arg in temp_args:

/usr/local/spark/python/pyspark/sql/utils.pyc in deco(_a, *_kw)
61 def deco(_a, *_kw):
62 try:
---> 63 return f(_a, *_kw)
64 except py4j.protocol.Py4JJavaError as e:
65 s = e.java_exception.toString()

/usr/local/spark/python/lib/py4j-0.10.1-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
310 raise Py4JJavaError(
311 "An error occurred while calling {0}{1}{2}.\n".
--> 312 format(target_id, ".", name), value)
313 else:
314 raise Py4JError(

Py4JJavaError: An error occurred while calling o41.loadClass.
: java.lang.NoClassDefFoundError: org/apache/spark/Logging
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:449)
at java.net.URLClassLoader.access$100(URLClassLoader.java:71)
at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:280)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:128)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:211)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.ClassNotFoundException: org.apache.spark.Logging
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 22 more