From 9b3c744a07498c60d1342c938b17af327a9c9001 Mon Sep 17 00:00:00 2001 From: xiejiajun Date: Fri, 20 Mar 2020 22:00:05 +0800 Subject: [PATCH 1/5] added timeout for getting Thrift client to avoid situations where the interpreter may not be restarted when the interpreter process exits unexpectedly --- .../zeppelin/interpreter/remote/RemoteInterpreterProcess.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 441dc76c931..de6b1574c22 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -48,7 +48,7 @@ public synchronized Client getClient() throws Exception { clientFactory = new ClientFactory(getHost(), getPort()); clientPool = new GenericObjectPool<>(clientFactory); } - return clientPool.borrowObject(); + return clientPool.borrowObject(5_000); } public void shutdown() { From 9bc56056b968c935e5f315060bc42cd59cac0b8f Mon Sep 17 00:00:00 2001 From: xiejiajun Date: Wed, 1 Apr 2020 01:43:50 +0800 Subject: [PATCH 2/5] bug fix: when we removing a paragraph , an interpreter process refused to connect due to it abnormal stop , which eventually caused the remove paragraph operation to fail to complete. --- .../InterpreterSettingManager.java | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index 85ab17eb8a1..899c850942d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -676,20 +676,23 @@ public List call(RemoteInterpreterService.Client client) throws Exceptio if (paragraphId != null) { resourceSet = resourceSet.filterByParagraphId(paragraphId); } - - for (final Resource r : resourceSet) { - remoteInterpreterProcess.callRemoteFunction( - new RemoteInterpreterProcess.RemoteFunction() { - - @Override - public Void call(RemoteInterpreterService.Client client) throws Exception { - client.resourceRemove( - r.getResourceId().getNoteId(), - r.getResourceId().getParagraphId(), - r.getResourceId().getName()); - return null; - } - }); + try{ + for (final Resource r : resourceSet) { + remoteInterpreterProcess.callRemoteFunction( + new RemoteInterpreterProcess.RemoteFunction() { + + @Override + public Void call(RemoteInterpreterService.Client client) throws Exception { + client.resourceRemove( + r.getResourceId().getNoteId(), + r.getResourceId().getParagraphId(), + r.getResourceId().getName()); + return null; + } + }); + } + }catch (Exception e){ + LOGGER.error(e.getMessage()); } } } From 6a05b4f8322407e84f1632834725c06109b10320 Mon Sep 17 00:00:00 2001 From: xiejiajun Date: Mon, 13 Jul 2020 17:51:38 +0800 Subject: [PATCH 3/5] =?UTF-8?q?Support=20for=20manually=20specifying=20the?= =?UTF-8?q?=20Java=20version=20of=20Spark=20Interpreter=20Scala=20REPL=20?= =?UTF-8?q?=C2=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/apache/zeppelin/spark/SparkScala210Interpreter.scala | 3 +++ .../org/apache/zeppelin/spark/SparkScala211Interpreter.scala | 2 ++ .../org/apache/zeppelin/spark/SparkScala212Interpreter.scala | 3 +++ 3 files changed, 8 insertions(+) diff --git a/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala b/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala index 0eac20096a6..f59f13722a1 100644 --- a/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala +++ b/spark/scala-2.10/src/main/scala/org/apache/zeppelin/spark/SparkScala210Interpreter.scala @@ -67,10 +67,13 @@ class SparkScala210Interpreter(override val conf: SparkConf, sparkHttpServer = server conf.set("spark.repl.class.uri", uri) } + val target = conf.get("spark.repl.target", "jvm-1.8") val settings = new Settings() settings.embeddedDefaults(sparkInterpreterClassLoader) settings.usejavacp.value = true + settings.target.value = target + this.userJars = getUserJars() LOGGER.info("UserJars: " + userJars.mkString(File.pathSeparator)) settings.classpath.value = userJars.mkString(File.pathSeparator) diff --git a/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala b/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala index cb5a016d40d..d2fb971bb62 100644 --- a/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala +++ b/spark/scala-2.11/src/main/scala/org/apache/zeppelin/spark/SparkScala211Interpreter.scala @@ -66,12 +66,14 @@ class SparkScala211Interpreter(override val conf: SparkConf, sparkHttpServer = server conf.set("spark.repl.class.uri", uri) } + val target = conf.get("spark.repl.target", "jvm-1.8") val settings = new Settings() settings.processArguments(List("-Yrepl-class-based", "-Yrepl-outdir", s"${outputDir.getAbsolutePath}"), true) settings.embeddedDefaults(sparkInterpreterClassLoader) settings.usejavacp.value = true + settings.target.value = target this.userJars = getUserJars() LOGGER.info("UserJars: " + userJars.mkString(File.pathSeparator)) diff --git a/spark/scala-2.12/src/main/scala/org/apache/zeppelin/spark/SparkScala212Interpreter.scala b/spark/scala-2.12/src/main/scala/org/apache/zeppelin/spark/SparkScala212Interpreter.scala index 2b04a1d360b..7f3512522f2 100644 --- a/spark/scala-2.12/src/main/scala/org/apache/zeppelin/spark/SparkScala212Interpreter.scala +++ b/spark/scala-2.12/src/main/scala/org/apache/zeppelin/spark/SparkScala212Interpreter.scala @@ -60,12 +60,15 @@ class SparkScala212Interpreter(override val conf: SparkConf, LOGGER.info("Scala shell repl output dir: " + outputDir.getAbsolutePath) outputDir.deleteOnExit() conf.set("spark.repl.class.outputDir", outputDir.getAbsolutePath) + val target = conf.get("spark.repl.target", "jvm-1.8") val settings = new Settings() settings.processArguments(List("-Yrepl-class-based", "-Yrepl-outdir", s"${outputDir.getAbsolutePath}"), true) settings.embeddedDefaults(sparkInterpreterClassLoader) settings.usejavacp.value = true + settings.target.value = target + this.userJars = getUserJars() LOGGER.info("UserJars: " + userJars.mkString(File.pathSeparator)) settings.classpath.value = userJars.mkString(File.pathSeparator) From 68720543a2ddf071cc4352a857bd830b75a84b71 Mon Sep 17 00:00:00 2001 From: xiejiajun Date: Mon, 13 Jul 2020 18:48:28 +0800 Subject: [PATCH 4/5] add spark.repl.target to spark.md docs --- docs/interpreter/spark.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index 5fc93056cd3..ca3683ddffe 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -199,6 +199,10 @@ You can also set other Spark properties which are not listed in the table. For a false whether use yarn proxy url as spark weburl, e.g. http://localhost:8088/proxy/application_1583396598068_0004 + spark.repl.target + jvm-1.8 + Manually specifying the Java version of Spark Interpreter Scala REPL,Available options:[jvm-1.5, jvm-1.6, jvm-1.7, jvm-1.8 + Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps. From d131f19e085ec54402471b2466cc33571208da58 Mon Sep 17 00:00:00 2001 From: xiejiajun Date: Mon, 13 Jul 2020 18:54:56 +0800 Subject: [PATCH 5/5] spark.md docs fix --- docs/interpreter/spark.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index ca3683ddffe..21069b43026 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -201,7 +201,7 @@ You can also set other Spark properties which are not listed in the table. For a spark.repl.target jvm-1.8 - Manually specifying the Java version of Spark Interpreter Scala REPL,Available options:[jvm-1.5, jvm-1.6, jvm-1.7, jvm-1.8 + Manually specifying the Java version of Spark Interpreter Scala REPL,Available options:[jvm-1.5, jvm-1.6, jvm-1.7, jvm-1.8]