From 50717dd340082545c1194134cca91d19169f97b6 Mon Sep 17 00:00:00 2001 From: Kris Geusebroek Date: Fri, 29 Apr 2016 17:24:34 +0200 Subject: [PATCH 1/5] Use slf4j instead of parquet bundled one since parquet doesn't bundle it anymore --- spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java index 93a3bcb5899..0ff07403911 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java @@ -22,8 +22,8 @@ import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterOutputListener; -import parquet.org.slf4j.Logger; -import parquet.org.slf4j.LoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.*; import java.util.Collections; From 24ea5843c7408bfd73b5c496533c38249f2e9720 Mon Sep 17 00:00:00 2001 From: Kris Geusebroek Date: Fri, 29 Apr 2016 17:24:48 +0200 Subject: [PATCH 2/5] method classServerUri not available in cdh5.7.0 Spark version. Only set config if variable is filled --- .../java/org/apache/zeppelin/spark/SparkInterpreter.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 7d134ee1c9c..0d99ba2ad9d 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -276,15 +276,18 @@ public SparkContext createSparkContext() { classServerUri = (String) classServer.invoke(interpreter.intp()); } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - throw new InterpreterException(e); + // continue instead of: throw new InterpreterException(e); } } SparkConf conf = new SparkConf() .setMaster(getProperty("master")) - .setAppName(getProperty("spark.app.name")) - .set("spark.repl.class.uri", classServerUri); + .setAppName(getProperty("spark.app.name")); + + if (classServerUri != null) { + conf.set("spark.repl.class.uri", classServerUri); + } if (jars.length > 0) { conf.setJars(jars); From 146b524ea9dac87733f7cfa4c7d1df6e82e917f6 Mon Sep 17 00:00:00 2001 From: Kris Geusebroek Date: Fri, 29 Apr 2016 17:25:08 +0200 Subject: [PATCH 3/5] Don't use extra driver classpath option since I use SPARK_CLASSPATH in spark-env.sh and using both is not supported --- bin/interpreter.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 17c9028b853..7804ca77cb4 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -152,7 +152,8 @@ addJarInDir "${LOCAL_INTERPRETER_REPO}" CLASSPATH+=":${ZEPPELIN_CLASSPATH}" if [[ -n "${SPARK_SUBMIT}" ]]; then - ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & + #${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & + ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & else ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & fi From 488cce681ee29d8296402c5fc9fcb130d8531dff Mon Sep 17 00:00:00 2001 From: Kris Geusebroek Date: Tue, 10 May 2016 10:14:26 +0200 Subject: [PATCH 4/5] Added logging and comments to clarify reason not throwing exception --- .../main/java/org/apache/zeppelin/spark/SparkInterpreter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 0d99ba2ad9d..60613d3d5ab 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -277,6 +277,9 @@ public SparkContext createSparkContext() { } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { // continue instead of: throw new InterpreterException(e); + // Newer Spark versions (like the patched CDH5.7.0 one) don't contain this method + logger.warn(String.format("Spark method classServerUri not available due to: [%s]", + e.getMessage())); } } From e33d5200a4e06d463712034778d1452e9d4bf316 Mon Sep 17 00:00:00 2001 From: Kris Geusebroek Date: Fri, 13 May 2016 08:38:11 +0200 Subject: [PATCH 5/5] Revert "Don't use extra driver classpath option since I use SPARK_CLASSPATH in spark-env.sh and using both is not supported" This reverts commit 146b524ea9dac87733f7cfa4c7d1df6e82e917f6. --- bin/interpreter.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 7804ca77cb4..17c9028b853 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -152,8 +152,7 @@ addJarInDir "${LOCAL_INTERPRETER_REPO}" CLASSPATH+=":${ZEPPELIN_CLASSPATH}" if [[ -n "${SPARK_SUBMIT}" ]]; then - #${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & - ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & + ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} & else ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} & fi