From 82eaefa41ed047f24ae5bdd133edea5036643247 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Wed, 24 Feb 2016 20:46:46 -0800 Subject: [PATCH 01/31] [ZEPPELIN-605] Add support for Scala 2.11 Enable Zeppelin to be built with both Scala 2.10 and Scala 2.11, mostly to start supporting interpreters that are moving to Scala 2.11 only such as Spark. --- README.md | 8 ++++ cassandra/pom.xml | 3 +- flink/pom.xml | 31 ++++++------- .../zeppelin/flink/FlinkInterpreter.java | 42 ++++++++++++++--- ignite/pom.xml | 10 ++--- pom.xml | 35 ++++++++++++--- scalding/pom.xml | 16 +++---- spark-dependencies/pom.xml | 10 ++++- spark/pom.xml | 12 ++++- zeppelin-display/pom.xml | 22 ++++++--- zeppelin-distribution/pom.xml | 45 +++++++++++++++++++ zeppelin-server/pom.xml | 27 ++++++++--- 12 files changed, 205 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index eee427cee3a..5feab28399b 100644 --- a/README.md +++ b/README.md @@ -294,6 +294,14 @@ And browse [localhost:8080](localhost:8080) in your browser. For configuration details check __`./conf`__ subdirectory. +### Building for Scala 2.11 + +To produce a Zeppelin package compiled with Scala 2.11, use the -Dscala-2.11 property: + +``` +mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Dscala-2.11 -DskipTests clean install +``` + ### Package To package the final distribution including the compressed archive, run: diff --git a/cassandra/pom.xml b/cassandra/pom.xml index 5ca86709791..c3e90c2f792 100644 --- a/cassandra/pom.xml +++ b/cassandra/pom.xml @@ -38,8 +38,6 @@ 3.0.1 1.0.5.4 1.3.0 - 2.10.4 - 2.10 3.3.2 1.7.1 16.0.1 @@ -173,6 +171,7 @@ org.scala-tools maven-scala-plugin + 2.15.2 compile diff --git a/flink/pom.xml b/flink/pom.xml index 226dc12a100..7355141fa66 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -37,8 +37,6 @@ 1.0.3 2.3.7 - 2.10 - 2.10.4 2.0.1 @@ -73,68 +71,71 @@ org.apache.flink - flink-clients_${flink.scala.binary.version} + flink-clients_${scala.binary.version} ${flink.version} org.apache.flink - flink-runtime_${flink.scala.binary.version} + flink-runtime_${scala.binary.version} ${flink.version} org.apache.flink - flink-scala_${flink.scala.binary.version} + flink-scala_${scala.binary.version} ${flink.version} org.apache.flink - flink-scala-shell_${flink.scala.binary.version} + flink-scala-shell_${scala.binary.version} ${flink.version} com.typesafe.akka - akka-actor_${flink.scala.binary.version} + akka-actor_${scala.binary.version} ${flink.akka.version} com.typesafe.akka - akka-remote_${flink.scala.binary.version} + akka-remote_${scala.binary.version} ${flink.akka.version} com.typesafe.akka - akka-slf4j_${flink.scala.binary.version} + akka-slf4j_${scala.binary.version} ${flink.akka.version} com.typesafe.akka - akka-testkit_${flink.scala.binary.version} + akka-testkit_${scala.binary.version} ${flink.akka.version} org.scala-lang scala-library - ${flink.scala.version} + ${scala.version} + provided org.scala-lang scala-compiler - ${flink.scala.version} + ${scala.version} + provided org.scala-lang scala-reflect - ${flink.scala.version} + ${scala.version} + provided @@ -169,7 +170,7 @@ net.alchim31.maven scala-maven-plugin - 3.1.4 + 3.2.2 @@ -199,7 +200,7 @@ org.scalamacros - paradise_${flink.scala.version} + paradise_${scala.version} ${scala.macros.version} diff --git a/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java b/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java index 68591d79754..18457700706 100644 --- a/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java +++ b/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java @@ -17,6 +17,7 @@ */ package org.apache.zeppelin.flink; +import java.lang.reflect.InvocationTargetException; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; @@ -24,10 +25,7 @@ import java.io.PrintWriter; import java.net.URL; import java.net.URLClassLoader; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import org.apache.flink.api.scala.FlinkILoop; import org.apache.flink.configuration.Configuration; @@ -45,6 +43,7 @@ import scala.Console; import scala.None; import scala.Some; +import scala.collection.immutable.Nil; import scala.runtime.AbstractFunction0; import scala.tools.nsc.Settings; import scala.tools.nsc.interpreter.IMain; @@ -103,7 +102,40 @@ public void open() { imain.interpret("import org.apache.flink.api.scala._"); imain.interpret("import org.apache.flink.api.common.functions._"); - imain.bindValue("env", env); + + String scalaVersion = scalaVersion(imain); + // scala 2.10 use imain.bindValue("env" env) + // scala 2.11 use imain.put("env", env); + String bindMethod = "bindValue"; + if (scalaVersion.equals("2.11")) { + bindMethod = "put"; + } + + java.lang.reflect.Method method; + try { + method = imain.getClass().getMethod(bindMethod, String.class, Object.class); + if (method != null) { + method.invoke(imain, "env", env); + } + } catch (Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("Error binding environment variable: " + e.getMessage(), e); + } + } + } + + private String scalaVersion(IMain imain) { + String version = "2.10"; + + try { + if (imain.getClass().getMethod("put", String.class, Object.class) != null) { + version = "2.11"; + } + } catch (Exception e) { + // ignore + } + + return version; } private boolean localMode() { diff --git a/ignite/pom.xml b/ignite/pom.xml index a3312a65953..66e6765aa7e 100644 --- a/ignite/pom.xml +++ b/ignite/pom.xml @@ -33,9 +33,7 @@ http://zeppelin.apache.org - 1.6.0 - 2.10 - 2.10.4 + 1.5.0.final @@ -73,19 +71,19 @@ org.scala-lang scala-library - ${ignite.scala.version} + ${scala.version} org.scala-lang scala-compiler - ${ignite.scala.version} + ${scala.version} org.scala-lang scala-reflect - ${ignite.scala.version} + ${scala.version} diff --git a/pom.xml b/pom.xml index 607fce91874..f6244485b35 100755 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,7 @@ --> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.maven-v4_0_0.xsd"> 4.0.0 @@ -79,6 +80,9 @@ + 2.10.5 + 2.10 + 1.7.10 1.2.17 0.9.2 @@ -136,7 +140,6 @@ 2.5 - com.google.code.gson gson @@ -155,14 +158,12 @@ 1.5 - commons-io commons-io 2.4 - commons-collections commons-collections @@ -181,7 +182,6 @@ ${guava.version} - junit junit @@ -412,7 +412,7 @@ .github/* .gitignore .repository/ - .Rhistory + .Rhistory **/*.diff **/*.patch **/*.avsc @@ -635,6 +635,30 @@ + + scala-2.10 + + !scala-2.11 + + + 2.10.5 + 2.10 + + + + + + scala-2.11 + + scala-2.11 + + + 2.11.7 + 2.11 + + + vendor-repo @@ -710,7 +734,6 @@ false - diff --git a/scalding/pom.xml b/scalding/pom.xml index e287f7b5c39..c561732551d 100644 --- a/scalding/pom.xml +++ b/scalding/pom.xml @@ -34,7 +34,6 @@ http://zeppelin.apache.org - 2.11.8 2.6.0 0.16.1-RC1 @@ -74,43 +73,43 @@ com.twitter - scalding-core_2.11 + scalding-core_${scala.binary.version} ${scalding.version} com.twitter - scalding-args_2.11 + scalding-args_${scala.binary.version} ${scalding.version} com.twitter - scalding-date_2.11 + scalding-date_${scala.binary.version} ${scalding.version} com.twitter - scalding-commons_2.11 + scalding-commons_${scala.binary.version} ${scalding.version} com.twitter - scalding-avro_2.11 + scalding-avro_${scala.binary.version} ${scalding.version} com.twitter - scalding-parquet_2.11 + scalding-parquet_${scala.binary.version} ${scalding.version} com.twitter - scalding-repl_2.11 + scalding-repl_${scala.binary.version} ${scalding.version} @@ -199,6 +198,7 @@ org.scala-tools maven-scala-plugin + 2.15.2 compile diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml index cb101b64df1..c2edf68b347 100644 --- a/spark-dependencies/pom.xml +++ b/spark-dependencies/pom.xml @@ -37,8 +37,6 @@ 1.4.1 - 2.10.4 - 2.10 2.3.0 ${hadoop.version} @@ -346,6 +344,14 @@ + + scala-2.11 + + 1.6.1 + http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz + + + spark-1.1 diff --git a/spark/pom.xml b/spark/pom.xml index 3a8a0feb530..18324f9dedd 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -39,10 +39,17 @@ 1.10.19 1.6.4 1.6.2 - 2.10.4 - 2.10 + + + scala-2.11 + + 1.6.2 + + + + org.slf4j @@ -412,6 +419,7 @@ org.scala-tools maven-scala-plugin + 2.15.2 compile diff --git a/zeppelin-display/pom.xml b/zeppelin-display/pom.xml index 4bf29e65e3a..2ed032db26c 100644 --- a/zeppelin-display/pom.xml +++ b/zeppelin-display/pom.xml @@ -33,11 +33,6 @@ Zeppelin: Display system apis http://zeppelin.apache.org - - 2.10.4 - 2.10 - - @@ -96,6 +91,23 @@ + + + scala-2.11 + + scala-2.11 + + + + + org.scala-lang.modules + scala-xml_${scala.binary.version} + 1.0.3 + + + + + diff --git a/zeppelin-distribution/pom.xml b/zeppelin-distribution/pom.xml index 65fc0cbbaa6..f65044e6faf 100644 --- a/zeppelin-distribution/pom.xml +++ b/zeppelin-distribution/pom.xml @@ -45,6 +45,34 @@ + + + + org.scala-lang + scala-library + ${scala.version} + + + + org.scala-lang + scala-compiler + ${scala.version} + + + + org.scala-lang + scala-reflect + ${scala.version} + + + + org.scala-lang + scalap + ${scala.version} + + + + zeppelin-server @@ -84,6 +112,23 @@ + + scala-2.11 + + scala-2.11 + + + + + + org.scala-lang.modules + scala-xml_${scala.binary.version} + 1.0.2 + + + + + publish-distr diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index fd5c31cd7f5..2ad1c87681e 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -43,19 +43,19 @@ org.scala-lang scala-library - 2.10.4 + ${scala.version} org.scala-lang scala-compiler - 2.10.4 + ${scala.version} org.scala-lang scalap - 2.10.4 + ${scala.version} @@ -258,8 +258,8 @@ org.scalatest - scalatest_2.10 - 2.1.1 + scalatest_${scala.binary.version} + 2.2.4 test @@ -393,6 +393,23 @@ + + scala-2.11 + + scala-2.11 + + + + + + org.scala-lang.modules + scala-xml_${scala.binary.version} + 1.0.2 + + + + + using-source-tree From 175be7a9a3b0358bb8868c5a59f20309d2328247 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Sat, 2 Apr 2016 10:14:11 -0700 Subject: [PATCH 02/31] [ZEPPELIN-605] Add Scala 2.11 build profile --- .travis.yml | 8 ++++++-- cassandra/pom.xml | 1 - pom.xml | 2 ++ r/pom.xml | 33 +++++++++++++++++++++++++++++---- spark/pom.xml | 17 +++++++---------- zeppelin-display/pom.xml | 6 +++--- zeppelin-server/pom.xml | 21 ++++++++++++++++++++- 7 files changed, 67 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index eec3b7cb2c1..091f32a6fe1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,9 +33,13 @@ addons: matrix: include: - # Test all modules + # Test all modules with scala 2.10 - jdk: "oraclejdk7" - env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="-Dpython.test.exclude=''" + env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Dscala-2.10 -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="-Dpython.test.exclude=''" + + # Test all modules with scala 2.11 + - jdk: "oraclejdk7" + env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Dscala-2.11 -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="-Dpython.test.exclude=''" # Test spark module for 1.5.2 - jdk: "oraclejdk7" diff --git a/cassandra/pom.xml b/cassandra/pom.xml index c3e90c2f792..dc071e3f86a 100644 --- a/cassandra/pom.xml +++ b/cassandra/pom.xml @@ -43,7 +43,6 @@ 16.0.1 - 2.2.4 4.12 3.2.4-Zeppelin 1.7.0 diff --git a/pom.xml b/pom.xml index f6244485b35..fce68ee1f78 100755 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,8 @@ 2.10.5 2.10 + 2.2.4 + 1.12.5 1.7.10 1.2.17 diff --git a/r/pom.xml b/r/pom.xml index e6c0403381d..b14424b0d21 100644 --- a/r/pom.xml +++ b/r/pom.xml @@ -36,8 +36,6 @@ .sh / 1.4.1 - 2.10.4 - 2.10 @@ -118,13 +116,13 @@ org.scalatest scalatest_${scala.binary.version} - 2.2.4 + ${scalatest.version} test org.scalacheck scalacheck_${scala.binary.version} - 1.12.5 + ${scalacheck.version} test @@ -376,4 +374,31 @@ + + + + + scala-2.10 + + !scala-2.11 + + + 1.6.1 + src/main/scala-2.10 + src/test/scala-2.10 + + + + + scala-2.11 + + scala-2.11 + + + 1.6.1 + src/main/scala-2.11 + src/test/scala/scala-2.11 + + + diff --git a/spark/pom.xml b/spark/pom.xml index 18324f9dedd..86879dbc53a 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -41,15 +41,6 @@ 1.6.2 - - - scala-2.11 - - 1.6.2 - - - - org.slf4j @@ -257,7 +248,7 @@ org.scalatest scalatest_${scala.binary.version} - 2.2.4 + ${scalatest.version} test @@ -448,6 +439,12 @@ + + scala-2.11 + + 1.6.2 + + diff --git a/zeppelin-display/pom.xml b/zeppelin-display/pom.xml index 2ed032db26c..1f316fcd5f7 100644 --- a/zeppelin-display/pom.xml +++ b/zeppelin-display/pom.xml @@ -85,8 +85,8 @@ org.scalatest - scalatest_2.10 - 2.1.1 + scalatest_${scala.binary.version} + ${scalatest.version} test @@ -102,7 +102,7 @@ org.scala-lang.modules scala-xml_${scala.binary.version} - 1.0.3 + 1.0.2 diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 2ad1c87681e..97b4c66268b 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -52,6 +52,12 @@ ${scala.version} + + org.scala-lang + scala-reflect + ${scala.version} + + org.scala-lang scalap @@ -221,6 +227,19 @@ org.scala-lang scala-library + ${scala.version} + + + + org.scala-lang + scala-compiler + ${scala.version} + + + + org.scala-lang + scala-reflect + ${scala.version} @@ -259,7 +278,7 @@ org.scalatest scalatest_${scala.binary.version} - 2.2.4 + ${scalatest.version} test From a73b68da3208e32de5eade7519df5a3f43d03549 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Thu, 7 Apr 2016 08:16:10 -0700 Subject: [PATCH 03/31] [ZEPPELIN-605] Enable Scala 2.11 REPL support for Spark Interpreter Scala 2.11 has changed a lot of it's REPL support and in order to enable building with Scala 2.10 and 2.11 we need to enable the new Scala 2.11 REPL and also make the build to compile based on the scala version. --- spark/pom.xml | 55 + .../apache/zeppelin/spark/DepInterpreter.java | 0 .../zeppelin/spark/SparkInterpreter.java | 0 .../spark/dep/SparkDependencyResolver.java | 0 .../apache/zeppelin/spark/DepInterpreter.java | 323 ++++++ .../zeppelin/spark/SparkInterpreter.java | 1007 +++++++++++++++++ .../spark/dep/SparkDependencyResolver.java | 351 ++++++ 7 files changed, 1736 insertions(+) rename spark/src/main/{java => scala-2.10}/org/apache/zeppelin/spark/DepInterpreter.java (100%) rename spark/src/main/{java => scala-2.10}/org/apache/zeppelin/spark/SparkInterpreter.java (100%) rename spark/src/main/{java => scala-2.10}/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java (100%) create mode 100644 spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java create mode 100644 spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java create mode 100644 spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java diff --git a/spark/pom.xml b/spark/pom.xml index 86879dbc53a..a49854339c2 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -64,6 +64,12 @@ ${project.version} + + org.apache.spark + spark-repl_${scala.binary.version} + ${spark.version} + + org.apache.spark spark-repl_${scala.binary.version} @@ -406,6 +412,38 @@ + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + ${extra.source.dir} + + + + + add-test-sources + generate-test-sources + + add-test-source + + + + ${extra.testsource.dir} + + + + + + org.scala-tools @@ -439,10 +477,27 @@ + + scala-2.10 + + !scala-2.11 + + + 1.6.1 + src/main/scala-2.10 + src/test/scala-2.10 + + + scala-2.11 + + scala-2.11 + 1.6.2 + src/main/scala-2.11 + src/test/scala/scala-2.11 diff --git a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/scala-2.10/org/apache/zeppelin/spark/DepInterpreter.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java rename to spark/src/main/scala-2.10/org/apache/zeppelin/spark/DepInterpreter.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/scala-2.10/org/apache/zeppelin/spark/SparkInterpreter.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java rename to spark/src/main/scala-2.10/org/apache/zeppelin/spark/SparkInterpreter.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java b/spark/src/main/scala-2.10/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java similarity index 100% rename from spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java rename to spark/src/main/scala-2.10/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java diff --git a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java new file mode 100644 index 00000000000..8fb08124f61 --- /dev/null +++ b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.spark.repl.SparkILoop; +import org.apache.spark.repl.Main; +import org.apache.spark.repl.SparkJLineCompletion; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.WrappedInterpreter; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.spark.dep.SparkDependencyContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonatype.aether.resolution.ArtifactResolutionException; +import org.sonatype.aether.resolution.DependencyResolutionException; + +import scala.Console; +import scala.None; +import scala.Some; +import scala.tools.nsc.Settings; +import scala.tools.nsc.interpreter.Completion.Candidates; +import scala.tools.nsc.interpreter.Completion.ScalaCompleter; +import scala.tools.nsc.settings.MutableSettings.BooleanSetting; +import scala.tools.nsc.settings.MutableSettings.PathSetting; + + +/** + * DepInterpreter downloads dependencies and pass them when SparkInterpreter initialized. + * It extends SparkInterpreter but does not create sparkcontext + * + */ +public class DepInterpreter extends Interpreter { + + static { + Interpreter.register( + "dep", + "spark", + DepInterpreter.class.getName(), + new InterpreterPropertyBuilder() + .add("zeppelin.dep.localrepo", + getSystemDefault("ZEPPELIN_DEP_LOCALREPO", null, "local-repo"), + "local repository for dependency loader") + .add("zeppelin.dep.additionalRemoteRepository", + "spark-packages,http://dl.bintray.com/spark-packages/maven,false;", + "A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository.") + .build()); + + } + + private Main intp; + private ByteArrayOutputStream out; + private SparkDependencyContext depc; + private SparkJLineCompletion completor; + private SparkILoop interpreter; + static final Logger LOGGER = LoggerFactory.getLogger(DepInterpreter.class); + + public DepInterpreter(Properties property) { + super(property); + } + + public SparkDependencyContext getDependencyContext() { + return depc; + } + + public static String getSystemDefault( + String envName, + String propertyName, + String defaultValue) { + + if (envName != null && !envName.isEmpty()) { + String envValue = System.getenv().get(envName); + if (envValue != null) { + return envValue; + } + } + + if (propertyName != null && !propertyName.isEmpty()) { + String propValue = System.getProperty(propertyName); + if (propValue != null) { + return propValue; + } + } + return defaultValue; + } + + @Override + public void close() { + if (intp != null) { + intp.close(); + } + } + + @Override + public void open() { + out = new ByteArrayOutputStream(); + createIMain(); + } + + + private void createIMain() { + Settings settings = new Settings(); + URL[] urls = getClassloaderUrls(); + + // set classpath for scala compiler + PathSetting pathSettings = settings.classpath(); + String classpath = ""; + List paths = currentClassPath(); + for (File f : paths) { + if (classpath.length() > 0) { + classpath += File.pathSeparator; + } + classpath += f.getAbsolutePath(); + } + + if (urls != null) { + for (URL u : urls) { + if (classpath.length() > 0) { + classpath += File.pathSeparator; + } + classpath += u.getFile(); + } + } + + pathSettings.v_$eq(classpath); + settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings); + + // set classloader for scala compiler + settings.explicitParentLoader_$eq(new Some(Thread.currentThread() + .getContextClassLoader())); + + BooleanSetting b = (BooleanSetting) settings.usejavacp(); + b.v_$eq(true); + settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b); + + interpreter = new SparkILoop(null, new PrintWriter(out)); + interpreter.settings_$eq(settings); + + interpreter.createInterpreter(); + + + intp = interpreter.intp(); + intp.setContextClassLoader(); + intp.initializeSynchronous(); + + depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"), + getProperty("zeppelin.dep.additionalRemoteRepository")); + completor = new SparkJLineCompletion(intp); + intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); + Map binder = (Map) getValue("_binder"); + binder.put("depc", depc); + + intp.interpret("@transient val z = " + + "_binder.get(\"depc\")" + + ".asInstanceOf[org.apache.zeppelin.spark.dep.SparkDependencyContext]"); + + } + + public Object getValue(String name) { + Object ret = intp.valueOfTerm(name); + if (ret instanceof None) { + return null; + } else if (ret instanceof Some) { + return ((Some) ret).get(); + } else { + return ret; + } + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + PrintStream printStream = new PrintStream(out); + Console.setOut(printStream); + out.reset(); + + SparkInterpreter sparkInterpreter = getSparkInterpreter(); + + if (sparkInterpreter != null && sparkInterpreter.isSparkContextInitialized()) { + return new InterpreterResult(Code.ERROR, + "Must be used before SparkInterpreter (%spark) initialized\n" + + "Hint: put this paragraph before any Spark code and " + + "restart Zeppelin/Interpreter" ); + } + + scala.tools.nsc.interpreter.Results.Result ret = intp.interpret(st); + Code code = getResultCode(ret); + + try { + depc.fetch(); + } catch (MalformedURLException | DependencyResolutionException + | ArtifactResolutionException e) { + LOGGER.error("Exception in DepInterpreter while interpret ", e); + return new InterpreterResult(Code.ERROR, e.toString()); + } + + if (code == Code.INCOMPLETE) { + return new InterpreterResult(code, "Incomplete expression"); + } else if (code == Code.ERROR) { + return new InterpreterResult(code, out.toString()); + } else { + return new InterpreterResult(code, out.toString()); + } + } + + private Code getResultCode(scala.tools.nsc.interpreter.Results.Result r) { + if (r instanceof scala.tools.nsc.interpreter.Results.Success$) { + return Code.SUCCESS; + } else if (r instanceof scala.tools.nsc.interpreter.Results.Incomplete$) { + return Code.INCOMPLETE; + } else { + return Code.ERROR; + } + } + + @Override + public void cancel(InterpreterContext context) { + } + + + @Override + public FormType getFormType() { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public List completion(String buf, int cursor) { + ScalaCompleter c = completor.completer(); + Candidates ret = c.complete(buf, cursor); + return scala.collection.JavaConversions.seqAsJavaList(ret.candidates()); + } + + private List currentClassPath() { + List paths = classPath(Thread.currentThread().getContextClassLoader()); + String[] cps = System.getProperty("java.class.path").split(File.pathSeparator); + if (cps != null) { + for (String cp : cps) { + paths.add(new File(cp)); + } + } + return paths; + } + + private List classPath(ClassLoader cl) { + List paths = new LinkedList(); + if (cl == null) { + return paths; + } + + if (cl instanceof URLClassLoader) { + URLClassLoader ucl = (URLClassLoader) cl; + URL[] urls = ucl.getURLs(); + if (urls != null) { + for (URL url : urls) { + paths.add(new File(url.getFile())); + } + } + } + return paths; + } + + private SparkInterpreter getSparkInterpreter() { + InterpreterGroup intpGroup = getInterpreterGroup(); + if (intpGroup == null) { + return null; + } + + Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName()); + if (p == null) { + return null; + } + + while (p instanceof WrappedInterpreter) { + p = ((WrappedInterpreter) p).getInnerInterpreter(); + } + return (SparkInterpreter) p; + } + + @Override + public Scheduler getScheduler() { + SparkInterpreter sparkInterpreter = getSparkInterpreter(); + if (sparkInterpreter != null) { + return getSparkInterpreter().getScheduler(); + } else { + return null; + } + } +} diff --git a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java new file mode 100644 index 00000000000..2e67d5b4955 --- /dev/null +++ b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java @@ -0,0 +1,1007 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark; + +import java.io.File; +import java.io.PrintWriter; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +import com.google.common.base.Joiner; + +import org.apache.spark.HttpServer; +import org.apache.spark.SparkConf; +import org.apache.spark.SparkContext; +import org.apache.spark.SparkEnv; +import org.apache.spark.repl.SparkCommandLine; +import org.apache.spark.repl.SparkILoop; +import org.apache.spark.repl.Main; +import org.apache.spark.repl.SparkJLineCompletion; +import org.apache.spark.scheduler.ActiveJob; +import org.apache.spark.scheduler.DAGScheduler; +import org.apache.spark.scheduler.Pool; +import org.apache.spark.sql.SQLContext; +import org.apache.spark.ui.jobs.JobProgressListener; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.InterpreterUtils; +import org.apache.zeppelin.interpreter.WrappedInterpreter; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.apache.zeppelin.spark.dep.SparkDependencyContext; +import org.apache.zeppelin.spark.dep.SparkDependencyResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import scala.*; +import scala.Enumeration.Value; +import scala.collection.Iterator; +import scala.collection.JavaConversions; +import scala.collection.JavaConverters; +import scala.collection.Seq; +import scala.collection.mutable.HashMap; +import scala.collection.mutable.HashSet; +import scala.reflect.io.AbstractFile; +import scala.tools.nsc.Settings; +import scala.tools.nsc.interpreter.Completion.Candidates; +import scala.tools.nsc.interpreter.Completion.ScalaCompleter; +import scala.tools.nsc.settings.MutableSettings.BooleanSetting; +import scala.tools.nsc.settings.MutableSettings.PathSetting; + +/** + * Spark interpreter for Zeppelin. + * + */ +public class SparkInterpreter extends Interpreter { + public static Logger logger = LoggerFactory.getLogger(SparkInterpreter.class); + + static { + Interpreter.register( + "spark", + "spark", + SparkInterpreter.class.getName(), + new InterpreterPropertyBuilder() + .add("spark.app.name", + getSystemDefault("SPARK_APP_NAME", "spark.app.name", "Zeppelin"), + "The name of spark application.") + .add("master", + getSystemDefault("MASTER", "spark.master", "local[*]"), + "Spark master uri. ex) spark://masterhost:7077") + .add("spark.executor.memory", + getSystemDefault(null, "spark.executor.memory", ""), + "Executor memory per worker instance. ex) 512m, 32g") + .add("spark.cores.max", + getSystemDefault(null, "spark.cores.max", ""), + "Total number of cores to use. Empty value uses all available core.") + .add("zeppelin.spark.useHiveContext", + getSystemDefault("ZEPPELIN_SPARK_USEHIVECONTEXT", + "zeppelin.spark.useHiveContext", "true"), + "Use HiveContext instead of SQLContext if it is true.") + .add("zeppelin.spark.maxResult", + getSystemDefault("ZEPPELIN_SPARK_MAXRESULT", "zeppelin.spark.maxResult", "1000"), + "Max number of SparkSQL result to display.") + .add("args", "", "spark commandline args") + .add("zeppelin.spark.printREPLOutput", "true", + "Print REPL output") + .build() + ); + } + + private ZeppelinContext z; + private SparkILoop interpreter; + private Main intp; + private static SparkContext sc; + private static SQLContext sqlc; + private static SparkEnv env; + private static JobProgressListener sparkListener; + private static AbstractFile classOutputDir; + private static Integer sharedInterpreterLock = new Integer(0); + private static AtomicInteger numReferenceOfSparkContext = new AtomicInteger(0); + + private SparkOutputStream out; + private SparkDependencyResolver dep; + private SparkJLineCompletion completor; + + private Map binder; + private SparkVersion sparkVersion; + + + public SparkInterpreter(Properties property) { + super(property); + out = new SparkOutputStream(); + } + + public SparkInterpreter(Properties property, SparkContext sc) { + this(property); + + this.sc = sc; + env = SparkEnv.get(); + sparkListener = setupListeners(this.sc); + } + + public SparkContext getSparkContext() { + synchronized (sharedInterpreterLock) { + if (sc == null) { + sc = createSparkContext(); + env = SparkEnv.get(); + sparkListener = setupListeners(sc); + } + return sc; + } + } + + public boolean isSparkContextInitialized() { + synchronized (sharedInterpreterLock) { + return sc != null; + } + } + + static JobProgressListener setupListeners(SparkContext context) { + JobProgressListener pl = new JobProgressListener(context.getConf()); + try { + Object listenerBus = context.getClass().getMethod("listenerBus").invoke(context); + + Method[] methods = listenerBus.getClass().getMethods(); + Method addListenerMethod = null; + for (Method m : methods) { + if (!m.getName().equals("addListener")) { + continue; + } + + Class[] parameterTypes = m.getParameterTypes(); + + if (parameterTypes.length != 1) { + continue; + } + + if (!parameterTypes[0].isAssignableFrom(JobProgressListener.class)) { + continue; + } + + addListenerMethod = m; + break; + } + + if (addListenerMethod != null) { + addListenerMethod.invoke(listenerBus, pl); + } else { + return null; + } + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + logger.error(e.toString(), e); + return null; + } + return pl; + } + + private boolean useHiveContext() { + return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.useHiveContext")); + } + + public SQLContext getSQLContext() { + synchronized (sharedInterpreterLock) { + if (sqlc == null) { + if (useHiveContext()) { + String name = "org.apache.spark.sql.hive.HiveContext"; + Constructor hc; + try { + hc = getClass().getClassLoader().loadClass(name) + .getConstructor(SparkContext.class); + sqlc = (SQLContext) hc.newInstance(getSparkContext()); + } catch (NoSuchMethodException | SecurityException + | ClassNotFoundException | InstantiationException + | IllegalAccessException | IllegalArgumentException + | InvocationTargetException e) { + logger.warn("Can't create HiveContext. Fallback to SQLContext", e); + // when hive dependency is not loaded, it'll fail. + // in this case SQLContext can be used. + sqlc = new SQLContext(getSparkContext()); + } + } else { + sqlc = new SQLContext(getSparkContext()); + } + } + return sqlc; + } + } + + public SparkDependencyResolver getDependencyResolver() { + if (dep == null) { + dep = new SparkDependencyResolver(intp, + sc, + getProperty("zeppelin.dep.localrepo"), + getProperty("zeppelin.dep.additionalRemoteRepository")); + } + return dep; + } + + private DepInterpreter getDepInterpreter() { + Interpreter p = getInterpreterInTheSameSessionByClassName(DepInterpreter.class.getName()); + if (p == null) { + return null; + } + + while (p instanceof WrappedInterpreter) { + p = ((WrappedInterpreter) p).getInnerInterpreter(); + } + return (DepInterpreter) p; + } + + public SparkContext createSparkContext() { + System.err.println("------ Create new SparkContext " + getProperty("master") + " -------"); + + String execUri = System.getenv("SPARK_EXECUTOR_URI"); + String[] jars = SparkILoop.getAddedJars(); + + String classServerUri = null; + + try { // in case of spark 1.1x, spark 1.2x + Method classServer = interpreter.intp().getClass().getMethod("classServer"); + HttpServer httpServer = (HttpServer) classServer.invoke(interpreter.intp()); + classServerUri = httpServer.uri(); + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + // continue + } + + if (classServerUri == null) { + try { // for spark 1.3x + Method classServer = interpreter.intp().getClass().getMethod("classServerUri"); + classServerUri = (String) classServer.invoke(interpreter.intp()); + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + throw new InterpreterException(e); + } + } + + SparkConf conf = + new SparkConf() + .setMaster(getProperty("master")) + .setAppName(getProperty("spark.app.name")) + .set("spark.repl.class.uri", classServerUri); + + if (jars.length > 0) { + conf.setJars(jars); + } + + if (execUri != null) { + conf.set("spark.executor.uri", execUri); + } + if (System.getenv("SPARK_HOME") != null) { + conf.setSparkHome(System.getenv("SPARK_HOME")); + } + conf.set("spark.scheduler.mode", "FAIR"); + + Properties intpProperty = getProperty(); + + for (Object k : intpProperty.keySet()) { + String key = (String) k; + String val = toString(intpProperty.get(key)); + if (!key.startsWith("spark.") || !val.trim().isEmpty()) { + logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val)); + conf.set(key, val); + } + } + + //TODO(jongyoul): Move these codes into PySparkInterpreter.java + String pysparkBasePath = getSystemDefault("SPARK_HOME", null, null); + File pysparkPath; + if (null == pysparkBasePath) { + pysparkBasePath = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../"); + pysparkPath = new File(pysparkBasePath, + "interpreter" + File.separator + "spark" + File.separator + "pyspark"); + } else { + pysparkPath = new File(pysparkBasePath, + "python" + File.separator + "lib"); + } + + //Only one of py4j-0.9-src.zip and py4j-0.8.2.1-src.zip should exist + String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip"}; + ArrayList pythonLibUris = new ArrayList<>(); + for (String lib : pythonLibs) { + File libFile = new File(pysparkPath, lib); + if (libFile.exists()) { + pythonLibUris.add(libFile.toURI().toString()); + } + } + pythonLibUris.trimToSize(); + + // Distribute two libraries(pyspark.zip and py4j-*.zip) to workers + // when spark version is less than or equal to 1.4.1 + if (pythonLibUris.size() == 2) { + try { + String confValue = conf.get("spark.yarn.dist.files"); + conf.set("spark.yarn.dist.files", confValue + "," + Joiner.on(",").join(pythonLibUris)); + } catch (NoSuchElementException e) { + conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris)); + } + if (!useSparkSubmit()) { + conf.set("spark.files", conf.get("spark.yarn.dist.files")); + } + conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs)); + } + + // Distributes needed libraries to workers + // when spark version is greater than or equal to 1.5.0 + if (getProperty("master").equals("yarn-client")) { + conf.set("spark.yarn.isPython", "true"); + } + + SparkContext sparkContext = new SparkContext(conf); + return sparkContext; + } + + static final String toString(Object o) { + return (o instanceof String) ? (String) o : ""; + } + + private boolean useSparkSubmit() { + return null != System.getenv("SPARK_SUBMIT"); + } + + public static String getSystemDefault( + String envName, + String propertyName, + String defaultValue) { + + if (envName != null && !envName.isEmpty()) { + String envValue = System.getenv().get(envName); + if (envValue != null) { + return envValue; + } + } + + if (propertyName != null && !propertyName.isEmpty()) { + String propValue = System.getProperty(propertyName); + if (propValue != null) { + return propValue; + } + } + return defaultValue; + } + + public boolean printREPLOutput() { + return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.printREPLOutput")); + } + + @Override + public void open() { + URL[] urls = getClassloaderUrls(); + + // Very nice discussion about how scala compiler handle classpath + // https://groups.google.com/forum/#!topic/scala-user/MlVwo2xCCI0 + + /* + * > val env = new nsc.Settings(errLogger) > env.usejavacp.value = true > val p = new + * Interpreter(env) > p.setContextClassLoader > Alternatively you can set the class path through + * nsc.Settings.classpath. + * + * >> val settings = new Settings() >> settings.usejavacp.value = true >> + * settings.classpath.value += File.pathSeparator + >> System.getProperty("java.class.path") >> + * val in = new Interpreter(settings) { >> override protected def parentClassLoader = + * getClass.getClassLoader >> } >> in.setContextClassLoader() + */ + Settings settings = new Settings(); + if (getProperty("args") != null) { + String[] argsArray = getProperty("args").split(" "); + LinkedList argList = new LinkedList(); + for (String arg : argsArray) { + argList.add(arg); + } + + SparkCommandLine command = + new SparkCommandLine(scala.collection.JavaConversions.asScalaBuffer( + argList).toList()); + settings = command.settings(); + } + + // set classpath for scala compiler + PathSetting pathSettings = settings.classpath(); + String classpath = ""; + List paths = currentClassPath(); + for (File f : paths) { + if (classpath.length() > 0) { + classpath += File.pathSeparator; + } + classpath += f.getAbsolutePath(); + } + + if (urls != null) { + for (URL u : urls) { + if (classpath.length() > 0) { + classpath += File.pathSeparator; + } + classpath += u.getFile(); + } + } + + // add dependency from DepInterpreter + DepInterpreter depInterpreter = getDepInterpreter(); + if (depInterpreter != null) { + SparkDependencyContext depc = depInterpreter.getDependencyContext(); + if (depc != null) { + List files = depc.getFiles(); + if (files != null) { + for (File f : files) { + if (classpath.length() > 0) { + classpath += File.pathSeparator; + } + classpath += f.getAbsolutePath(); + } + } + } + } + + // add dependency from local repo + String localRepo = getProperty("zeppelin.interpreter.localRepo"); + if (localRepo != null) { + File localRepoDir = new File(localRepo); + if (localRepoDir.exists()) { + File[] files = localRepoDir.listFiles(); + if (files != null) { + for (File f : files) { + if (classpath.length() > 0) { + classpath += File.pathSeparator; + } + classpath += f.getAbsolutePath(); + } + } + } + } + + pathSettings.v_$eq(classpath); + settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings); + + + // set classloader for scala compiler + settings.explicitParentLoader_$eq(new Some(Thread.currentThread() + .getContextClassLoader())); + BooleanSetting b = (BooleanSetting) settings.usejavacp(); + b.v_$eq(true); + settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b); + + System.setProperty("scala.repl.name.line", "line" + this.hashCode() + "$"); + + synchronized (sharedInterpreterLock) { + /* create scala repl */ + if (printREPLOutput()) { + this.interpreter = new SparkILoop(null, new PrintWriter(out)); + } else { + this.interpreter = new SparkILoop(null, new PrintWriter(Console.out(), false)); + } + + interpreter.settings_$eq(settings); + + interpreter.createInterpreter(); + + intp = interpreter.intp(); + intp.setContextClassLoader(); + intp.initializeSynchronous(); + + if (classOutputDir == null) { + classOutputDir = settings.outputDirs().getSingleOutput().get(); + } else { + // change Main class output dir + settings.outputDirs().setSingleOutput(classOutputDir); + ClassLoader cl = intp.classLoader(); + + try { + Field rootField = cl.getClass().getSuperclass().getDeclaredField("root"); + rootField.setAccessible(true); + rootField.set(cl, classOutputDir); + } catch (NoSuchFieldException | IllegalAccessException e) { + logger.error(e.getMessage(), e); + } + } + + completor = new SparkJLineCompletion(intp); + + sc = getSparkContext(); + if (sc.getPoolForName("fair").isEmpty()) { + Value schedulingMode = org.apache.spark.scheduler.SchedulingMode.FAIR(); + int minimumShare = 0; + int weight = 1; + Pool pool = new Pool("fair", schedulingMode, minimumShare, weight); + sc.taskScheduler().rootPool().addSchedulable(pool); + } + + sparkVersion = SparkVersion.fromVersionString(sc.version()); + + sqlc = getSQLContext(); + + dep = getDependencyResolver(); + + z = new ZeppelinContext(sc, sqlc, null, dep, + Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); + + intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); + binder = (Map) getValue("_binder"); + binder.put("sc", sc); + binder.put("sqlc", sqlc); + binder.put("z", z); + + intp.interpret("@transient val z = " + + "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]"); + intp.interpret("@transient val sc = " + + "_binder.get(\"sc\").asInstanceOf[org.apache.spark.SparkContext]"); + intp.interpret("@transient val sqlc = " + + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); + intp.interpret("@transient val sqlContext = " + + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); + intp.interpret("import org.apache.spark.SparkContext._"); + + if (sparkVersion.oldSqlContextImplicits()) { + intp.interpret("import sqlContext._"); + } else { + intp.interpret("import sqlContext.implicits._"); + intp.interpret("import sqlContext.sql"); + intp.interpret("import org.apache.spark.sql.functions._"); + } + } + + /* Temporary disabling DisplayUtils. see https://issues.apache.org/jira/browse/ZEPPELIN-127 + * + // Utility functions for display + intp.interpret("import org.apache.zeppelin.spark.utils.DisplayUtils._"); + + // Scala implicit value for spark.maxResult + intp.interpret("import org.apache.zeppelin.spark.utils.SparkMaxResult"); + intp.interpret("implicit val sparkMaxResult = new SparkMaxResult(" + + Integer.parseInt(getProperty("zeppelin.spark.maxResult")) + ")"); + */ + + try { + if (sparkVersion.oldLoadFilesMethodName()) { + Method loadFiles = this.interpreter.getClass().getMethod("loadFiles", Settings.class); + loadFiles.invoke(this.interpreter, settings); + } else { + Method loadFiles = this.interpreter.getClass().getMethod( + "org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class); + loadFiles.invoke(this.interpreter, settings); + } + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + throw new InterpreterException(e); + } + + // add jar from DepInterpreter + if (depInterpreter != null) { + SparkDependencyContext depc = depInterpreter.getDependencyContext(); + if (depc != null) { + List files = depc.getFilesDist(); + if (files != null) { + for (File f : files) { + if (f.getName().toLowerCase().endsWith(".jar")) { + sc.addJar(f.getAbsolutePath()); + logger.info("sc.addJar(" + f.getAbsolutePath() + ")"); + } else { + sc.addFile(f.getAbsolutePath()); + logger.info("sc.addFile(" + f.getAbsolutePath() + ")"); + } + } + } + } + } + + // add jar from local repo + if (localRepo != null) { + File localRepoDir = new File(localRepo); + if (localRepoDir.exists()) { + File[] files = localRepoDir.listFiles(); + if (files != null) { + for (File f : files) { + if (f.getName().toLowerCase().endsWith(".jar")) { + sc.addJar(f.getAbsolutePath()); + logger.info("sc.addJar(" + f.getAbsolutePath() + ")"); + } else { + sc.addFile(f.getAbsolutePath()); + logger.info("sc.addFile(" + f.getAbsolutePath() + ")"); + } + } + } + } + } + + numReferenceOfSparkContext.incrementAndGet(); + } + + private List currentClassPath() { + List paths = classPath(Thread.currentThread().getContextClassLoader()); + String[] cps = System.getProperty("java.class.path").split(File.pathSeparator); + if (cps != null) { + for (String cp : cps) { + paths.add(new File(cp)); + } + } + return paths; + } + + private List classPath(ClassLoader cl) { + List paths = new LinkedList(); + if (cl == null) { + return paths; + } + + if (cl instanceof URLClassLoader) { + URLClassLoader ucl = (URLClassLoader) cl; + URL[] urls = ucl.getURLs(); + if (urls != null) { + for (URL url : urls) { + paths.add(new File(url.getFile())); + } + } + } + return paths; + } + + @Override + public List completion(String buf, int cursor) { + if (buf.length() < cursor) { + cursor = buf.length(); + } + String completionText = getCompletionTargetString(buf, cursor); + if (completionText == null) { + completionText = ""; + cursor = completionText.length(); + } + ScalaCompleter c = completor.completer(); + Candidates ret = c.complete(completionText, cursor); + return scala.collection.JavaConversions.seqAsJavaList(ret.candidates()); + } + + private String getCompletionTargetString(String text, int cursor) { + String[] completionSeqCharaters = {" ", "\n", "\t"}; + int completionEndPosition = cursor; + int completionStartPosition = cursor; + int indexOfReverseSeqPostion = cursor; + + String resultCompletionText = ""; + String completionScriptText = ""; + try { + completionScriptText = text.substring(0, cursor); + } + catch (Exception e) { + logger.error(e.toString()); + return null; + } + completionEndPosition = completionScriptText.length(); + + String tempReverseCompletionText = new StringBuilder(completionScriptText).reverse().toString(); + + for (String seqCharacter : completionSeqCharaters) { + indexOfReverseSeqPostion = tempReverseCompletionText.indexOf(seqCharacter); + + if (indexOfReverseSeqPostion < completionStartPosition && indexOfReverseSeqPostion > 0) { + completionStartPosition = indexOfReverseSeqPostion; + } + + } + + if (completionStartPosition == completionEndPosition) { + completionStartPosition = 0; + } + else + { + completionStartPosition = completionEndPosition - completionStartPosition; + } + resultCompletionText = completionScriptText.substring( + completionStartPosition , completionEndPosition); + + return resultCompletionText; + } + + public Object getValue(String name) { + Object ret = intp.valueOfTerm(name); + if (ret instanceof None) { + return null; + } else if (ret instanceof Some) { + return ((Some) ret).get(); + } else { + return ret; + } + } + + String getJobGroup(InterpreterContext context){ + return "zeppelin-" + context.getParagraphId(); + } + + /** + * Interpret a single line. + */ + @Override + public InterpreterResult interpret(String line, InterpreterContext context) { + if (sparkVersion.isUnsupportedVersion()) { + return new InterpreterResult(Code.ERROR, "Spark " + sparkVersion.toString() + + " is not supported"); + } + + z.setInterpreterContext(context); + if (line == null || line.trim().length() == 0) { + return new InterpreterResult(Code.SUCCESS); + } + return interpret(line.split("\n"), context); + } + + public InterpreterResult interpret(String[] lines, InterpreterContext context) { + synchronized (this) { + z.setGui(context.getGui()); + sc.setJobGroup(getJobGroup(context), "Zeppelin", false); + InterpreterResult r = interpretInput(lines, context); + sc.clearJobGroup(); + return r; + } + } + + public InterpreterResult interpretInput(String[] lines, InterpreterContext context) { + SparkEnv.set(env); + + // add print("") to make sure not finishing with comment + // see https://github.com/NFLabs/zeppelin/issues/151 + String[] linesToRun = new String[lines.length + 1]; + for (int i = 0; i < lines.length; i++) { + linesToRun[i] = lines[i]; + } + linesToRun[lines.length] = "print(\"\")"; + + Console.setOut(context.out); + out.setInterpreterOutput(context.out); + context.out.clear(); + Code r = null; + String incomplete = ""; + boolean inComment = false; + + for (int l = 0; l < linesToRun.length; l++) { + String s = linesToRun[l]; + // check if next line starts with "." (but not ".." or "./") it is treated as an invocation + if (l + 1 < linesToRun.length) { + String nextLine = linesToRun[l + 1].trim(); + boolean continuation = false; + if (nextLine.isEmpty() + || nextLine.startsWith("//") // skip empty line or comment + || nextLine.startsWith("}") + || nextLine.startsWith("object")) { // include "} object" for Scala companion object + continuation = true; + } else if (!inComment && nextLine.startsWith("/*")) { + inComment = true; + continuation = true; + } else if (inComment && nextLine.lastIndexOf("*/") >= 0) { + inComment = false; + continuation = true; + } else if (nextLine.length() > 1 + && nextLine.charAt(0) == '.' + && nextLine.charAt(1) != '.' // ".." + && nextLine.charAt(1) != '/') { // "./" + continuation = true; + } else if (inComment) { + continuation = true; + } + if (continuation) { + incomplete += s + "\n"; + continue; + } + } + + scala.tools.nsc.interpreter.Results.Result res = null; + try { + res = intp.interpret(incomplete + s); + } catch (Exception e) { + sc.clearJobGroup(); + out.setInterpreterOutput(null); + logger.info("Interpreter exception", e); + return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e)); + } + + r = getResultCode(res); + + if (r == Code.ERROR) { + sc.clearJobGroup(); + out.setInterpreterOutput(null); + return new InterpreterResult(r, ""); + } else if (r == Code.INCOMPLETE) { + incomplete += s + "\n"; + } else { + incomplete = ""; + } + } + + if (r == Code.INCOMPLETE) { + sc.clearJobGroup(); + out.setInterpreterOutput(null); + return new InterpreterResult(r, "Incomplete expression"); + } else { + sc.clearJobGroup(); + out.setInterpreterOutput(null); + return new InterpreterResult(Code.SUCCESS); + } + } + + + @Override + public void cancel(InterpreterContext context) { + sc.cancelJobGroup(getJobGroup(context)); + } + + @Override + public int getProgress(InterpreterContext context) { + String jobGroup = getJobGroup(context); + int completedTasks = 0; + int totalTasks = 0; + + DAGScheduler scheduler = sc.dagScheduler(); + if (scheduler == null) { + return 0; + } + HashSet jobs = scheduler.activeJobs(); + if (jobs == null || jobs.size() == 0) { + return 0; + } + Iterator it = jobs.iterator(); + while (it.hasNext()) { + ActiveJob job = it.next(); + String g = (String) job.properties().get("spark.jobGroup.id"); + if (jobGroup.equals(g)) { + int[] progressInfo = null; + try { + Object finalStage = job.getClass().getMethod("finalStage").invoke(job); + if (sparkVersion.getProgress1_0()) { + progressInfo = getProgressFromStage_1_0x(sparkListener, finalStage); + } else { + progressInfo = getProgressFromStage_1_1x(sparkListener, finalStage); + } + } catch (IllegalAccessException | IllegalArgumentException + | InvocationTargetException | NoSuchMethodException + | SecurityException e) { + logger.error("Can't get progress info", e); + return 0; + } + totalTasks += progressInfo[0]; + completedTasks += progressInfo[1]; + } + } + + if (totalTasks == 0) { + return 0; + } + return completedTasks * 100 / totalTasks; + } + + private int[] getProgressFromStage_1_0x(JobProgressListener sparkListener, Object stage) + throws IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException { + int numTasks = (int) stage.getClass().getMethod("numTasks").invoke(stage); + int completedTasks = 0; + + int id = (int) stage.getClass().getMethod("id").invoke(stage); + + Object completedTaskInfo = null; + + completedTaskInfo = JavaConversions.mapAsJavaMap( + (HashMap) sparkListener.getClass() + .getMethod("stageIdToTasksComplete").invoke(sparkListener)).get(id); + + if (completedTaskInfo != null) { + completedTasks += (int) completedTaskInfo; + } + List parents = JavaConversions.seqAsJavaList((Seq) stage.getClass() + .getMethod("parents").invoke(stage)); + if (parents != null) { + for (Object s : parents) { + int[] p = getProgressFromStage_1_0x(sparkListener, s); + numTasks += p[0]; + completedTasks += p[1]; + } + } + + return new int[] {numTasks, completedTasks}; + } + + private int[] getProgressFromStage_1_1x(JobProgressListener sparkListener, Object stage) + throws IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException { + int numTasks = (int) stage.getClass().getMethod("numTasks").invoke(stage); + int completedTasks = 0; + int id = (int) stage.getClass().getMethod("id").invoke(stage); + + try { + Method stageIdToData = sparkListener.getClass().getMethod("stageIdToData"); + HashMap, Object> stageIdData = + (HashMap, Object>) stageIdToData.invoke(sparkListener); + Class stageUIDataClass = + this.getClass().forName("org.apache.spark.ui.jobs.UIData$StageUIData"); + + Method numCompletedTasks = stageUIDataClass.getMethod("numCompleteTasks"); + Set> keys = + JavaConverters.setAsJavaSetConverter(stageIdData.keySet()).asJava(); + for (Tuple2 k : keys) { + if (id == (int) k._1()) { + Object uiData = stageIdData.get(k).get(); + completedTasks += (int) numCompletedTasks.invoke(uiData); + } + } + } catch (Exception e) { + logger.error("Error on getting progress information", e); + } + + List parents = JavaConversions.seqAsJavaList((Seq) stage.getClass() + .getMethod("parents").invoke(stage)); + if (parents != null) { + for (Object s : parents) { + int[] p = getProgressFromStage_1_1x(sparkListener, s); + numTasks += p[0]; + completedTasks += p[1]; + } + } + return new int[] {numTasks, completedTasks}; + } + + private Code getResultCode(scala.tools.nsc.interpreter.Results.Result r) { + if (r instanceof scala.tools.nsc.interpreter.Results.Success$) { + return Code.SUCCESS; + } else if (r instanceof scala.tools.nsc.interpreter.Results.Incomplete$) { + return Code.INCOMPLETE; + } else { + return Code.ERROR; + } + } + + @Override + public void close() { + logger.info("Close interpreter"); + + if (numReferenceOfSparkContext.decrementAndGet() == 0) { + sc.stop(); + sc = null; + } + + intp.close(); + } + + @Override + public FormType getFormType() { + return FormType.NATIVE; + } + + public JobProgressListener getJobProgressListener() { + return sparkListener; + } + + @Override + public Scheduler getScheduler() { + return SchedulerFactory.singleton().createOrGetFIFOScheduler( + SparkInterpreter.class.getName() + this.hashCode()); + } + + public ZeppelinContext getZeppelinContext() { + return z; + } + + public SparkVersion getSparkVersion() { + return sparkVersion; + } +} diff --git a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java new file mode 100644 index 00000000000..4671c5cefc1 --- /dev/null +++ b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark.dep; + +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.spark.SparkContext; +import org.apache.spark.repl.Main; +import org.apache.zeppelin.dep.AbstractDependencyResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonatype.aether.artifact.Artifact; +import org.sonatype.aether.collection.CollectRequest; +import org.sonatype.aether.graph.Dependency; +import org.sonatype.aether.graph.DependencyFilter; +import org.sonatype.aether.repository.RemoteRepository; +import org.sonatype.aether.resolution.ArtifactResult; +import org.sonatype.aether.resolution.DependencyRequest; +import org.sonatype.aether.util.artifact.DefaultArtifact; +import org.sonatype.aether.util.artifact.JavaScopes; +import org.sonatype.aether.util.filter.DependencyFilterUtils; +import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter; + +import scala.Some; +import scala.collection.IndexedSeq; +import scala.reflect.io.AbstractFile; +import scala.tools.nsc.Global; +import scala.tools.nsc.backend.JavaPlatform; +import scala.tools.nsc.util.ClassPath; +import scala.tools.nsc.util.MergedClassPath; + +/** + * Deps resolver. + * Add new dependencies from mvn repo (at runtime) to Spark interpreter group. + */ +public class SparkDependencyResolver extends AbstractDependencyResolver { + Logger logger = LoggerFactory.getLogger(SparkDependencyResolver.class); + private Global global; + private Main intp; + private SparkContext sc; + + private final String[] exclusions = new String[] {"org.scala-lang:scala-library", + "org.scala-lang:scala-compiler", + "org.scala-lang:scala-reflect", + "org.scala-lang:scalap", + "org.apache.zeppelin:zeppelin-zengine", + "org.apache.zeppelin:zeppelin-spark", + "org.apache.zeppelin:zeppelin-server"}; + + public SparkDependencyResolver(Main intp, SparkContext sc, String localRepoPath, + String additionalRemoteRepository) { + super(localRepoPath); + this.intp = intp; + this.global = intp.global(); + this.sc = sc; + addRepoFromProperty(additionalRemoteRepository); + } + + private void addRepoFromProperty(String listOfRepo) { + if (listOfRepo != null) { + String[] repos = listOfRepo.split(";"); + for (String repo : repos) { + String[] parts = repo.split(","); + if (parts.length == 3) { + String id = parts[0].trim(); + String url = parts[1].trim(); + boolean isSnapshot = Boolean.parseBoolean(parts[2].trim()); + if (id.length() > 1 && url.length() > 1) { + addRepo(id, url, isSnapshot); + } + } + } + } + } + + private void updateCompilerClassPath(URL[] urls) throws IllegalAccessException, + IllegalArgumentException, InvocationTargetException { + + JavaPlatform platform = (JavaPlatform) global.platform(); + MergedClassPath newClassPath = mergeUrlsIntoClassPath(platform, urls); + + Method[] methods = platform.getClass().getMethods(); + for (Method m : methods) { + if (m.getName().endsWith("currentClassPath_$eq")) { + m.invoke(platform, new Some(newClassPath)); + break; + } + } + + // NOTE: Must use reflection until this is exposed/fixed upstream in Scala + List classPaths = new LinkedList(); + for (URL url : urls) { + classPaths.add(url.getPath()); + } + + // Reload all jars specified into our compiler + global.invalidateClassPathEntries(scala.collection.JavaConversions.asScalaBuffer(classPaths) + .toList()); + } + + // Until spark 1.1.x + // check https://github.com/apache/spark/commit/191d7cf2a655d032f160b9fa181730364681d0e7 + private void updateRuntimeClassPath_1_x(URL[] urls) throws SecurityException, + IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException { + ClassLoader cl = intp.classLoader().getParent(); + Method addURL; + addURL = cl.getClass().getDeclaredMethod("addURL", new Class[] {URL.class}); + addURL.setAccessible(true); + for (URL url : urls) { + addURL.invoke(cl, url); + } + } + + private void updateRuntimeClassPath_2_x(URL[] urls) throws SecurityException, + IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException { + ClassLoader cl = intp.classLoader().getParent(); + Method addURL; + addURL = cl.getClass().getDeclaredMethod("addNewUrl", new Class[] {URL.class}); + addURL.setAccessible(true); + for (URL url : urls) { + addURL.invoke(cl, url); + } + } + + private MergedClassPath mergeUrlsIntoClassPath(JavaPlatform platform, URL[] urls) { + IndexedSeq> entries = + ((MergedClassPath) platform.classPath()).entries(); + List> cp = new LinkedList>(); + + for (int i = 0; i < entries.size(); i++) { + cp.add(entries.apply(i)); + } + + for (URL url : urls) { + AbstractFile file; + if ("file".equals(url.getProtocol())) { + File f = new File(url.getPath()); + if (f.isDirectory()) { + file = AbstractFile.getDirectory(scala.reflect.io.File.jfile2path(f)); + } else { + file = AbstractFile.getFile(scala.reflect.io.File.jfile2path(f)); + } + } else { + file = AbstractFile.getURL(url); + } + + ClassPath newcp = platform.classPath().context().newClassPath(file); + + // distinct + if (cp.contains(newcp) == false) { + cp.add(newcp); + } + } + + return new MergedClassPath(scala.collection.JavaConversions.asScalaBuffer(cp).toIndexedSeq(), + platform.classPath().context()); + } + + public List load(String artifact, + boolean addSparkContext) throws Exception { + return load(artifact, new LinkedList(), addSparkContext); + } + + public List load(String artifact, Collection excludes, + boolean addSparkContext) throws Exception { + if (StringUtils.isBlank(artifact)) { + // Should throw here + throw new RuntimeException("Invalid artifact to load"); + } + + // :[:[:]]: + int numSplits = artifact.split(":").length; + if (numSplits >= 3 && numSplits <= 6) { + return loadFromMvn(artifact, excludes, addSparkContext); + } else { + loadFromFs(artifact, addSparkContext); + LinkedList libs = new LinkedList(); + libs.add(artifact); + return libs; + } + } + + private void loadFromFs(String artifact, boolean addSparkContext) throws Exception { + File jarFile = new File(artifact); + + intp.global().new Run(); + + if (sc.version().startsWith("1.1")) { + updateRuntimeClassPath_1_x(new URL[] {jarFile.toURI().toURL()}); + } else { + updateRuntimeClassPath_2_x(new URL[] {jarFile.toURI().toURL()}); + } + + if (addSparkContext) { + sc.addJar(jarFile.getAbsolutePath()); + } + } + + private List loadFromMvn(String artifact, Collection excludes, + boolean addSparkContext) throws Exception { + List loadedLibs = new LinkedList(); + Collection allExclusions = new LinkedList(); + allExclusions.addAll(excludes); + allExclusions.addAll(Arrays.asList(exclusions)); + + List listOfArtifact; + listOfArtifact = getArtifactsWithDep(artifact, allExclusions); + + Iterator it = listOfArtifact.iterator(); + while (it.hasNext()) { + Artifact a = it.next().getArtifact(); + String gav = a.getGroupId() + ":" + a.getArtifactId() + ":" + a.getVersion(); + for (String exclude : allExclusions) { + if (gav.startsWith(exclude)) { + it.remove(); + break; + } + } + } + + List newClassPathList = new LinkedList(); + List files = new LinkedList(); + for (ArtifactResult artifactResult : listOfArtifact) { + logger.info("Load " + artifactResult.getArtifact().getGroupId() + ":" + + artifactResult.getArtifact().getArtifactId() + ":" + + artifactResult.getArtifact().getVersion()); + newClassPathList.add(artifactResult.getArtifact().getFile().toURI().toURL()); + files.add(artifactResult.getArtifact().getFile()); + loadedLibs.add(artifactResult.getArtifact().getGroupId() + ":" + + artifactResult.getArtifact().getArtifactId() + ":" + + artifactResult.getArtifact().getVersion()); + } + + intp.global().new Run(); + if (sc.version().startsWith("1.1")) { + updateRuntimeClassPath_1_x(newClassPathList.toArray(new URL[0])); + } else { + updateRuntimeClassPath_2_x(newClassPathList.toArray(new URL[0])); + } + updateCompilerClassPath(newClassPathList.toArray(new URL[0])); + + if (addSparkContext) { + for (File f : files) { + sc.addJar(f.getAbsolutePath()); + } + } + + return loadedLibs; + } + + /** + * @param dependency + * @param excludes list of pattern can either be of the form groupId:artifactId + * @return + * @throws Exception + */ + @Override + public List getArtifactsWithDep(String dependency, + Collection excludes) throws Exception { + Artifact artifact = new DefaultArtifact(inferScalaVersion(dependency)); + DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE); + PatternExclusionsDependencyFilter exclusionFilter = + new PatternExclusionsDependencyFilter(inferScalaVersion(excludes)); + + CollectRequest collectRequest = new CollectRequest(); + collectRequest.setRoot(new Dependency(artifact, JavaScopes.COMPILE)); + + synchronized (repos) { + for (RemoteRepository repo : repos) { + collectRequest.addRepository(repo); + } + } + DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, + DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter)); + return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); + } + + public static Collection inferScalaVersion(Collection artifact) { + List list = new LinkedList(); + for (String a : artifact) { + list.add(inferScalaVersion(a)); + } + return list; + } + + public static String inferScalaVersion(String artifact) { + int pos = artifact.indexOf(":"); + if (pos < 0 || pos + 2 >= artifact.length()) { + // failed to infer + return artifact; + } + + if (':' == artifact.charAt(pos + 1)) { + String restOfthem = ""; + String versionSep = ":"; + + String groupId = artifact.substring(0, pos); + int nextPos = artifact.indexOf(":", pos + 2); + if (nextPos < 0) { + if (artifact.charAt(artifact.length() - 1) == '*') { + nextPos = artifact.length() - 1; + versionSep = ""; + restOfthem = "*"; + } else { + versionSep = ""; + nextPos = artifact.length(); + } + } + + String artifactId = artifact.substring(pos + 2, nextPos); + if (nextPos < artifact.length()) { + if (!restOfthem.equals("*")) { + restOfthem = artifact.substring(nextPos + 1); + } + } + + String [] version = scala.util.Properties.versionNumberString().split("[.]"); + String scalaVersion = version[0] + "." + version[1]; + + return groupId + ":" + artifactId + "_" + scalaVersion + versionSep + restOfthem; + } else { + return artifact; + } + } +} From 5c47d9a70104c4740a86e1c05d457642256d5467 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Tue, 12 Apr 2016 22:15:55 -0700 Subject: [PATCH 04/31] [ZEPPELIN-605] Rewrite Spark interpreter based on Scala 2.11 support --- .../apache/zeppelin/spark/DepInterpreter.java | 41 +++++----- .../zeppelin/spark/SparkInterpreter.java | 75 +++++++++---------- .../spark/dep/SparkDependencyResolver.java | 10 +-- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java index 8fb08124f61..ee101789752 100644 --- a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java +++ b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java @@ -17,21 +17,15 @@ package org.apache.zeppelin.spark; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintStream; -import java.io.PrintWriter; +import java.io.*; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import org.apache.spark.repl.SparkILoop; import org.apache.spark.repl.Main; -import org.apache.spark.repl.SparkJLineCompletion; +// import org.apache.spark.repl.SparkJLineCompletion; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -80,9 +74,10 @@ public class DepInterpreter extends Interpreter { } private Main intp; + private ByteArrayInputStream in; private ByteArrayOutputStream out; private SparkDependencyContext depc; - private SparkJLineCompletion completor; + // private SparkJLineCompletion completor; private SparkILoop interpreter; static final Logger LOGGER = LoggerFactory.getLogger(DepInterpreter.class); @@ -118,7 +113,7 @@ public static String getSystemDefault( @Override public void close() { if (intp != null) { - intp.close(); + intp.interp().interpreter().close(); } } @@ -164,31 +159,34 @@ private void createIMain() { b.v_$eq(true); settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b); - interpreter = new SparkILoop(null, new PrintWriter(out)); + interpreter = new SparkILoop((java.io.BufferedReader) null, new PrintWriter(out)); interpreter.settings_$eq(settings); interpreter.createInterpreter(); - intp = interpreter.intp(); - intp.setContextClassLoader(); - intp.initializeSynchronous(); + intp.interp_$eq(this.interpreter); + intp.interp().interpreter().setContextClassLoader(); + intp.interp().interpreter().initializeSynchronous(); + depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"), getProperty("zeppelin.dep.additionalRemoteRepository")); - completor = new SparkJLineCompletion(intp); - intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); + // TODO(lresende) Support for completion + // completor = new SparkJLineCompletion(intp); + intp.interp().interpreter().interpret("" + + "@transient var _binder = new java.util.HashMap[String, Object]()"); Map binder = (Map) getValue("_binder"); binder.put("depc", depc); - intp.interpret("@transient val z = " + intp.interp().interpreter().interpret("@transient val z = " + "_binder.get(\"depc\")" + ".asInstanceOf[org.apache.zeppelin.spark.dep.SparkDependencyContext]"); } public Object getValue(String name) { - Object ret = intp.valueOfTerm(name); + Object ret = intp.interp().interpreter().valueOfTerm(name); if (ret instanceof None) { return null; } else if (ret instanceof Some) { @@ -213,7 +211,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) { "restart Zeppelin/Interpreter" ); } - scala.tools.nsc.interpreter.Results.Result ret = intp.interpret(st); + scala.tools.nsc.interpreter.Results.Result ret = intp.interp().interpreter().interpret(st); Code code = getResultCode(ret); try { @@ -260,9 +258,12 @@ public int getProgress(InterpreterContext context) { @Override public List completion(String buf, int cursor) { + /* ScalaCompleter c = completor.completer(); Candidates ret = c.complete(buf, cursor); return scala.collection.JavaConversions.seqAsJavaList(ret.candidates()); + */ + return Collections.emptyList(); } private List currentClassPath() { diff --git a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java index 2e67d5b4955..7c694232c9d 100644 --- a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java @@ -34,10 +34,10 @@ import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.spark.SparkEnv; -import org.apache.spark.repl.SparkCommandLine; +// import org.apache.spark.repl.SparkCommandLine; import org.apache.spark.repl.SparkILoop; import org.apache.spark.repl.Main; -import org.apache.spark.repl.SparkJLineCompletion; +// import org.apache.spark.repl.SparkJLineCompletion; import org.apache.spark.scheduler.ActiveJob; import org.apache.spark.scheduler.DAGScheduler; import org.apache.spark.scheduler.Pool; @@ -125,7 +125,7 @@ public class SparkInterpreter extends Interpreter { private SparkOutputStream out; private SparkDependencyResolver dep; - private SparkJLineCompletion completor; + // private SparkJLineCompletion completor; private Map binder; private SparkVersion sparkVersion; @@ -257,7 +257,8 @@ public SparkContext createSparkContext() { System.err.println("------ Create new SparkContext " + getProperty("master") + " -------"); String execUri = System.getenv("SPARK_EXECUTOR_URI"); - String[] jars = SparkILoop.getAddedJars(); + // TODO(lresende) Figure out added jars ==>> SparkILoop.getAddedJars(); + String[] jars = new String[0]; String classServerUri = null; @@ -397,16 +398,6 @@ public void open() { // Very nice discussion about how scala compiler handle classpath // https://groups.google.com/forum/#!topic/scala-user/MlVwo2xCCI0 - /* - * > val env = new nsc.Settings(errLogger) > env.usejavacp.value = true > val p = new - * Interpreter(env) > p.setContextClassLoader > Alternatively you can set the class path through - * nsc.Settings.classpath. - * - * >> val settings = new Settings() >> settings.usejavacp.value = true >> - * settings.classpath.value += File.pathSeparator + >> System.getProperty("java.class.path") >> - * val in = new Interpreter(settings) { >> override protected def parentClassLoader = - * getClass.getClassLoader >> } >> in.setContextClassLoader() - */ Settings settings = new Settings(); if (getProperty("args") != null) { String[] argsArray = getProperty("args").split(" "); @@ -415,10 +406,11 @@ public void open() { argList.add(arg); } + /* SparkCommandLine command = - new SparkCommandLine(scala.collection.JavaConversions.asScalaBuffer( - argList).toList()); + new SparkCommandLine(scala.collection.JavaConversions.asScalaBuffer(argList).toList()); settings = command.settings(); + */ } // set classpath for scala compiler @@ -480,8 +472,8 @@ public void open() { // set classloader for scala compiler - settings.explicitParentLoader_$eq(new Some(Thread.currentThread() - .getContextClassLoader())); + settings.explicitParentLoader_$eq( + new Some(Thread.currentThread().getContextClassLoader())); BooleanSetting b = (BooleanSetting) settings.usejavacp(); b.v_$eq(true); settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b); @@ -491,25 +483,27 @@ public void open() { synchronized (sharedInterpreterLock) { /* create scala repl */ if (printREPLOutput()) { - this.interpreter = new SparkILoop(null, new PrintWriter(out)); + this.interpreter = new SparkILoop((java.io.BufferedReader) null, + new PrintWriter(out)); } else { - this.interpreter = new SparkILoop(null, new PrintWriter(Console.out(), false)); + this.interpreter = new SparkILoop((java.io.BufferedReader) null, + new PrintWriter(Console.out())); } interpreter.settings_$eq(settings); interpreter.createInterpreter(); - intp = interpreter.intp(); - intp.setContextClassLoader(); - intp.initializeSynchronous(); + intp.interp_$eq(this.interpreter); + intp.interp().interpreter().setContextClassLoader(); + intp.interp().interpreter().initializeSynchronous(); if (classOutputDir == null) { classOutputDir = settings.outputDirs().getSingleOutput().get(); } else { // change Main class output dir settings.outputDirs().setSingleOutput(classOutputDir); - ClassLoader cl = intp.classLoader(); + ClassLoader cl = intp.interp().interpreter().classLoader(); try { Field rootField = cl.getClass().getSuperclass().getDeclaredField("root"); @@ -520,7 +514,8 @@ public void open() { } } - completor = new SparkJLineCompletion(intp); + // TODO(lresende) support for completion + // completor = new SparkJLineCompletion(intp); sc = getSparkContext(); if (sc.getPoolForName("fair").isEmpty()) { @@ -540,28 +535,29 @@ public void open() { z = new ZeppelinContext(sc, sqlc, null, dep, Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); - intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); + intp.interp().interpreter().interpret("" + + "@transient var _binder = new java.util.HashMap[String, Object]()"); binder = (Map) getValue("_binder"); binder.put("sc", sc); binder.put("sqlc", sqlc); binder.put("z", z); - intp.interpret("@transient val z = " + intp.interp().interpreter().interpret("@transient val z = " + "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]"); - intp.interpret("@transient val sc = " + intp.interp().interpreter().interpret("@transient val sc = " + "_binder.get(\"sc\").asInstanceOf[org.apache.spark.SparkContext]"); - intp.interpret("@transient val sqlc = " + intp.interp().interpreter().interpret("@transient val sqlc = " + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); - intp.interpret("@transient val sqlContext = " + intp.interp().interpreter().interpret("@transient val sqlContext = " + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); - intp.interpret("import org.apache.spark.SparkContext._"); + intp.interp().interpreter().interpret("import org.apache.spark.SparkContext._"); if (sparkVersion.oldSqlContextImplicits()) { - intp.interpret("import sqlContext._"); + intp.interp().interpreter().interpret("import sqlContext._"); } else { - intp.interpret("import sqlContext.implicits._"); - intp.interpret("import sqlContext.sql"); - intp.interpret("import org.apache.spark.sql.functions._"); + intp.interp().interpreter().interpret("import sqlContext.implicits._"); + intp.interp().interpreter().interpret("import sqlContext.sql"); + intp.interp().interpreter().interpret("import org.apache.spark.sql.functions._"); } } @@ -662,6 +658,7 @@ private List classPath(ClassLoader cl) { @Override public List completion(String buf, int cursor) { + /* if (buf.length() < cursor) { cursor = buf.length(); } @@ -673,6 +670,8 @@ public List completion(String buf, int cursor) { ScalaCompleter c = completor.completer(); Candidates ret = c.complete(completionText, cursor); return scala.collection.JavaConversions.seqAsJavaList(ret.candidates()); + */ + return Collections.emptyList(); } private String getCompletionTargetString(String text, int cursor) { @@ -717,7 +716,7 @@ private String getCompletionTargetString(String text, int cursor) { } public Object getValue(String name) { - Object ret = intp.valueOfTerm(name); + Object ret = intp.interp().interpreter().valueOfTerm(name); if (ret instanceof None) { return null; } else if (ret instanceof Some) { @@ -809,7 +808,7 @@ public InterpreterResult interpretInput(String[] lines, InterpreterContext conte scala.tools.nsc.interpreter.Results.Result res = null; try { - res = intp.interpret(incomplete + s); + res = intp.interp().interpreter().interpret(incomplete + s); } catch (Exception e) { sc.clearJobGroup(); out.setInterpreterOutput(null); @@ -979,7 +978,7 @@ public void close() { sc = null; } - intp.close(); + intp.interp().interpreter().close(); } @Override diff --git a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java index 4671c5cefc1..9c00c556cef 100644 --- a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java +++ b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java @@ -75,7 +75,7 @@ public SparkDependencyResolver(Main intp, SparkContext sc, String localRepoPath, String additionalRemoteRepository) { super(localRepoPath); this.intp = intp; - this.global = intp.global(); + this.global = intp.interp().interpreter().global(); this.sc = sc; addRepoFromProperty(additionalRemoteRepository); } @@ -127,7 +127,7 @@ private void updateCompilerClassPath(URL[] urls) throws IllegalAccessException, private void updateRuntimeClassPath_1_x(URL[] urls) throws SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException { - ClassLoader cl = intp.classLoader().getParent(); + ClassLoader cl = intp.interp().interpreter().classLoader().getParent(); Method addURL; addURL = cl.getClass().getDeclaredMethod("addURL", new Class[] {URL.class}); addURL.setAccessible(true); @@ -139,7 +139,7 @@ private void updateRuntimeClassPath_1_x(URL[] urls) throws SecurityException, private void updateRuntimeClassPath_2_x(URL[] urls) throws SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException { - ClassLoader cl = intp.classLoader().getParent(); + ClassLoader cl = intp.interp().interpreter().classLoader().getParent(); Method addURL; addURL = cl.getClass().getDeclaredMethod("addNewUrl", new Class[] {URL.class}); addURL.setAccessible(true); @@ -209,7 +209,7 @@ public List load(String artifact, Collection excludes, private void loadFromFs(String artifact, boolean addSparkContext) throws Exception { File jarFile = new File(artifact); - intp.global().new Run(); + intp.interp().interpreter().global().new Run(); if (sc.version().startsWith("1.1")) { updateRuntimeClassPath_1_x(new URL[] {jarFile.toURI().toURL()}); @@ -257,7 +257,7 @@ private List loadFromMvn(String artifact, Collection excludes, + artifactResult.getArtifact().getVersion()); } - intp.global().new Run(); + intp.interp().interpreter().global().new Run(); if (sc.version().startsWith("1.1")) { updateRuntimeClassPath_1_x(newClassPathList.toArray(new URL[0])); } else { From c88348de33e4ff3786dd4185a81d134ccba36acb Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 9 Jun 2016 09:45:03 -0700 Subject: [PATCH 05/31] Initial scala-210, 211 support in the single binary --- spark/pom.xml | 62 - .../apache/zeppelin/spark/DepInterpreter.java | 156 ++- .../zeppelin/spark/PySparkInterpreter.java | 2 - .../zeppelin/spark/SparkInterpreter.java | 245 +++- .../zeppelin/spark/SparkSqlInterpreter.java | 2 - .../apache/zeppelin/spark/SparkVersion.java | 6 +- .../spark/dep/SparkDependencyResolver.java | 28 +- .../apache/zeppelin/spark/DepInterpreter.java | 324 ------ .../zeppelin/spark/SparkInterpreter.java | 1006 ----------------- .../spark/dep/SparkDependencyResolver.java | 351 ------ .../zeppelin/spark/SparkInterpreterTest.java | 2 - .../dep/SparkDependencyResolverTest.java | 1 - 12 files changed, 339 insertions(+), 1846 deletions(-) rename spark/src/main/{scala-2.10 => java}/org/apache/zeppelin/spark/DepInterpreter.java (66%) rename spark/src/main/{scala-2.10 => java}/org/apache/zeppelin/spark/SparkInterpreter.java (82%) rename spark/src/main/{scala-2.10 => java}/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java (94%) delete mode 100644 spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java delete mode 100644 spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java delete mode 100644 spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java diff --git a/spark/pom.xml b/spark/pom.xml index a49854339c2..ae2887ef6c1 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -64,12 +64,6 @@ ${project.version} - - org.apache.spark - spark-repl_${scala.binary.version} - ${spark.version} - - org.apache.spark spark-repl_${scala.binary.version} @@ -412,38 +406,6 @@ - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-source - generate-sources - - add-source - - - - ${extra.source.dir} - - - - - add-test-sources - generate-test-sources - - add-test-source - - - - ${extra.testsource.dir} - - - - - - org.scala-tools @@ -477,30 +439,6 @@ - - scala-2.10 - - !scala-2.11 - - - 1.6.1 - src/main/scala-2.10 - src/test/scala-2.10 - - - - - scala-2.11 - - scala-2.11 - - - 1.6.2 - src/main/scala-2.11 - src/test/scala/scala-2.11 - - - sparkr diff --git a/spark/src/main/scala-2.10/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java similarity index 66% rename from spark/src/main/scala-2.10/org/apache/zeppelin/spark/DepInterpreter.java rename to spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java index 28c588551f4..6e72d3af27a 100644 --- a/spark/src/main/scala-2.10/org/apache/zeppelin/spark/DepInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java @@ -21,21 +21,22 @@ import java.io.File; import java.io.PrintStream; import java.io.PrintWriter; -import java.lang.reflect.Type; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; -import java.util.*; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import org.apache.spark.repl.SparkILoop; -import org.apache.spark.repl.SparkIMain; -import org.apache.spark.repl.SparkJLineCompletion; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.WrappedInterpreter; @@ -54,6 +55,7 @@ import scala.tools.nsc.Settings; import scala.tools.nsc.interpreter.Completion.Candidates; import scala.tools.nsc.interpreter.Completion.ScalaCompleter; +import scala.tools.nsc.interpreter.Results; import scala.tools.nsc.settings.MutableSettings.BooleanSetting; import scala.tools.nsc.settings.MutableSettings.PathSetting; @@ -64,10 +66,17 @@ * */ public class DepInterpreter extends Interpreter { - private SparkIMain intp; + /** + * intp - org.apache.spark.repl.SparkIMain (scala 2.10) + * intp - scala.tools.nsc.interpreter.IMain; (scala 2.11) + */ + private Object intp; private ByteArrayOutputStream out; private SparkDependencyContext depc; - private SparkJLineCompletion completor; + /** + * completor - org.apache.spark.repl.SparkJLineCompletion (scala 2.10) + */ + private Object completor; private SparkILoop interpreter; static final Logger LOGGER = LoggerFactory.getLogger(DepInterpreter.class); @@ -103,7 +112,7 @@ public static String getSystemDefault( @Override public void close() { if (intp != null) { - intp.close(); + invokeMethod(intp, "close"); } } @@ -149,31 +158,47 @@ private void createIMain() { b.v_$eq(true); settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b); - interpreter = new SparkILoop(null, new PrintWriter(out)); + interpreter = new SparkILoop((java.io.BufferedReader) null, new PrintWriter(out)); interpreter.settings_$eq(settings); interpreter.createInterpreter(); intp = interpreter.intp(); - intp.setContextClassLoader(); - intp.initializeSynchronous(); + + if (isScala2_10()) { + invokeMethod(intp, "setContextClassLoader"); + invokeMethod(intp, "initializeSynchronous"); + } depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"), getProperty("zeppelin.dep.additionalRemoteRepository")); - completor = new SparkJLineCompletion(intp); - intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); + if (isScala2_10()) { + completor = instantiateClass( + "SparkJLineCompletion", + new Class[]{findClass("org.apache.spark.repl.SparkIMain")}, + new Object[]{intp}); + } + interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); Map binder = (Map) getValue("_binder"); binder.put("depc", depc); - intp.interpret("@transient val z = " + interpret("@transient val z = " + "_binder.get(\"depc\")" + ".asInstanceOf[org.apache.zeppelin.spark.dep.SparkDependencyContext]"); } + private Results.Result interpret(String line) { + return (Results.Result) invokeMethod( + intp, + "interpret", + new Class[] {String.class}, + new Object[] {line}); + } + public Object getValue(String name) { - Object ret = intp.valueOfTerm(name); + Object ret = invokeMethod(intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); if (ret instanceof None) { return null; } else if (ret instanceof Some) { @@ -198,7 +223,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) { "restart Zeppelin/Interpreter" ); } - scala.tools.nsc.interpreter.Results.Result ret = intp.interpret(st); + scala.tools.nsc.interpreter.Results.Result ret = interpret(st); Code code = getResultCode(ret); try { @@ -245,17 +270,22 @@ public int getProgress(InterpreterContext context) { @Override public List completion(String buf, int cursor) { - ScalaCompleter c = completor.completer(); - Candidates ret = c.complete(buf, cursor); + if (isScala2_10()) { + ScalaCompleter c = (ScalaCompleter) invokeMethod(completor, "completer"); + Candidates ret = c.complete(buf, cursor); - List candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates()); - List completions = new LinkedList(); + List candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates()); + List completions = new LinkedList(); - for (String candidate : candidates) { - completions.add(new InterpreterCompletion(candidate, candidate)); + for (String candidate : candidates) { + completions.add(new InterpreterCompletion(candidate, candidate)); + } + + return completions; + } else { + return new LinkedList(); } - return completions; } private List currentClassPath() { @@ -313,4 +343,84 @@ public Scheduler getScheduler() { return null; } } + + private Object invokeMethod(Object o, String name) { + return invokeMethod(o, name, new Class[]{}, new Object[]{}); + } + + private Object invokeMethod(Object o, String name, Class [] argTypes, Object [] params) { + try { + return o.getClass().getMethod(name, argTypes).invoke(o, params); + } catch (NoSuchMethodException e) { + logger.error(e.getMessage(), e); + } catch (InvocationTargetException e) { + logger.error(e.getMessage(), e); + } catch (IllegalAccessException e) { + logger.error(e.getMessage(), e); + } + + return null; + } + + private Object invokeStaticMethod(Class c, String name) { + return invokeStaticMethod(c, name, new Class[]{}, new Object[]{}); + } + + private Object invokeStaticMethod(Class c, String name, Class [] argTypes, Object [] params) { + try { + return c.getMethod(name, argTypes).invoke(null, params); + } catch (NoSuchMethodException e) { + logger.error(e.getMessage(), e); + } catch (InvocationTargetException e) { + logger.error(e.getMessage(), e); + e.printStackTrace(); + } catch (IllegalAccessException e) { + logger.error(e.getMessage(), e); + } + + return null; + } + + private Object instantiateClass(String name, Class [] argTypes, Object [] params) { + try { + Constructor constructor = getClass().getClassLoader() + .loadClass(name).getConstructor(argTypes); + return constructor.newInstance(params); + } catch (NoSuchMethodException e) { + logger.error(e.getMessage(), e); + } catch (ClassNotFoundException e) { + logger.error(e.getMessage(), e); + } catch (IllegalAccessException e) { + logger.error(e.getMessage(), e); + } catch (InstantiationException e) { + logger.error(e.getMessage(), e); + } catch (InvocationTargetException e) { + logger.error(e.getMessage(), e); + } + return null; + } + + // function works after intp is initialized + private boolean isScala2_10() { + try { + this.getClass().forName("org.apache.spark.repl.SparkIMain"); + return true; + } catch (ClassNotFoundException e) { + return false; + } + } + + private boolean isScala2_11() { + return !isScala2_11(); + } + + + private Class findClass(String name) { + try { + return this.getClass().forName(name); + } catch (ClassNotFoundException e) { + logger.error(e.getMessage(), e); + return null; + } + } } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index 43462ad8880..98fb8346ee0 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -48,8 +48,6 @@ import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; diff --git a/spark/src/main/scala-2.10/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java similarity index 82% rename from spark/src/main/scala-2.10/org/apache/zeppelin/spark/SparkInterpreter.java rename to spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 53bfc02b370..bce72245660 100644 --- a/spark/src/main/scala-2.10/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -19,10 +19,15 @@ import java.io.File; import java.io.PrintWriter; -import java.lang.reflect.*; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; import java.util.*; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import com.google.common.base.Joiner; @@ -33,10 +38,8 @@ import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.spark.SparkEnv; -import org.apache.spark.repl.SparkCommandLine; + import org.apache.spark.repl.SparkILoop; -import org.apache.spark.repl.SparkIMain; -import org.apache.spark.repl.SparkJLineCompletion; import org.apache.spark.scheduler.ActiveJob; import org.apache.spark.scheduler.DAGScheduler; import org.apache.spark.scheduler.Pool; @@ -45,7 +48,6 @@ import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterUtils; @@ -72,9 +74,11 @@ import scala.collection.mutable.HashMap; import scala.collection.mutable.HashSet; import scala.reflect.io.AbstractFile; +import scala.tools.nsc.Global; import scala.tools.nsc.Settings; import scala.tools.nsc.interpreter.Completion.Candidates; import scala.tools.nsc.interpreter.Completion.ScalaCompleter; +import scala.tools.nsc.interpreter.Results; import scala.tools.nsc.settings.MutableSettings; import scala.tools.nsc.settings.MutableSettings.BooleanSetting; import scala.tools.nsc.settings.MutableSettings.PathSetting; @@ -88,7 +92,11 @@ public class SparkInterpreter extends Interpreter { private ZeppelinContext z; private SparkILoop interpreter; - private SparkIMain intp; + /** + * intp - org.apache.spark.repl.SparkIMain (scala 2.10) + * intp - scala.tools.nsc.interpreter.IMain; (scala 2.11) + */ + private Object intp; private static SparkContext sc; private static SQLContext sqlc; private static SparkEnv env; @@ -99,7 +107,11 @@ public class SparkInterpreter extends Interpreter { private SparkOutputStream out; private SparkDependencyResolver dep; - private SparkJLineCompletion completor; + + /** + * completor - org.apache.spark.repl.SparkJLineCompletion (scala 2.10) + */ + private Object completor; private Map binder; private SparkVersion sparkVersion; @@ -209,12 +221,15 @@ public SQLContext getSQLContext() { } } + public SparkDependencyResolver getDependencyResolver() { if (dep == null) { - dep = new SparkDependencyResolver(intp, - sc, - getProperty("zeppelin.dep.localrepo"), - getProperty("zeppelin.dep.additionalRemoteRepository")); + dep = new SparkDependencyResolver( + (Global) invokeMethod(intp, "global"), + (ClassLoader) invokeMethod(invokeMethod(intp, "classLoader"), "getParent"), + sc, + getProperty("zeppelin.dep.localrepo"), + getProperty("zeppelin.dep.additionalRemoteRepository")); } return dep; } @@ -235,7 +250,13 @@ public SparkContext createSparkContext() { logger.info("------ Create new SparkContext {} -------", getProperty("master")); String execUri = System.getenv("SPARK_EXECUTOR_URI"); - String[] jars = SparkILoop.getAddedJars(); + String[] jars = null; + + if (isScala2_10()) { + jars = (String[]) invokeStaticMethod(SparkILoop.class, "getAddedJars"); + } else { + jars = (String[]) invokeStaticMethod(findClass("org.apache.spark.repl.Main"), "getAddedJars"); + } String classServerUri = null; @@ -399,10 +420,19 @@ public void open() { argList.add(arg); } - SparkCommandLine command = - new SparkCommandLine(scala.collection.JavaConversions.asScalaBuffer( - argList).toList()); - settings = command.settings(); + scala.collection.immutable.List list = + JavaConversions.asScalaBuffer(argList).toList(); + + if (isScala2_10()) { + Object sparkCommandLine = instantiateClass( + "org.apache.spark.repl.SparkCommandLine", + new Class[]{ list.getClass() }, + new Object[]{ list }); + + settings = (Settings) invokeMethod(sparkCommandLine, "settings"); + } else { + settings.processArguments(list, true); + } } // set classpath for scala compiler @@ -481,9 +511,10 @@ public void open() { synchronized (sharedInterpreterLock) { /* create scala repl */ if (printREPLOutput()) { - this.interpreter = new SparkILoop(null, new PrintWriter(out)); + this.interpreter = new SparkILoop((java.io.BufferedReader) null, new PrintWriter(out)); } else { - this.interpreter = new SparkILoop(null, new PrintWriter(Console.out(), false)); + this.interpreter = new SparkILoop((java.io.BufferedReader) null, + new PrintWriter(Console.out(), false)); } interpreter.settings_$eq(settings); @@ -491,16 +522,18 @@ public void open() { interpreter.createInterpreter(); intp = interpreter.intp(); - intp.setContextClassLoader(); - intp.initializeSynchronous(); + + if (isScala2_10()) { + invokeMethod(intp, "setContextClassLoader"); + invokeMethod(intp, "initializeSynchronous"); + } if (classOutputDir == null) { classOutputDir = settings.outputDirs().getSingleOutput().get(); } else { // change SparkIMain class output dir settings.outputDirs().setSingleOutput(classOutputDir); - ClassLoader cl = intp.classLoader(); - + ClassLoader cl = (ClassLoader) invokeMethod(intp, "classLoader"); try { Field rootField = cl.getClass().getSuperclass().getDeclaredField("root"); rootField.setAccessible(true); @@ -510,7 +543,12 @@ public void open() { } } - completor = new SparkJLineCompletion(intp); + if (isScala2_10()) { + completor = instantiateClass( + "SparkJLineCompletion", + new Class[]{findClass("org.apache.spark.repl.SparkIMain")}, + new Object[]{intp}); + } sc = getSparkContext(); if (sc.getPoolForName("fair").isEmpty()) { @@ -530,31 +568,29 @@ public void open() { z = new ZeppelinContext(sc, sqlc, null, dep, Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); - intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); + interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); binder = (Map) getValue("_binder"); binder.put("sc", sc); binder.put("sqlc", sqlc); binder.put("z", z); - binder.put("intp", intp); - intp.interpret("@transient val intp = _binder.get(\"intp\").asInstanceOf[org.apache.spark" + - ".repl.SparkIMain]"); - intp.interpret("@transient val z = " + + interpret("@transient val z = " + "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]"); - intp.interpret("@transient val sc = " + interpret("@transient val sc = " + "_binder.get(\"sc\").asInstanceOf[org.apache.spark.SparkContext]"); - intp.interpret("@transient val sqlc = " + interpret("@transient val sqlc = " + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); - intp.interpret("@transient val sqlContext = " + interpret("@transient val sqlContext = " + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); - intp.interpret("import org.apache.spark.SparkContext._"); + interpret("import org.apache.spark.SparkContext._"); if (importImplicit()) { if (sparkVersion.oldSqlContextImplicits()) { - intp.interpret("import sqlContext._"); + interpret("import sqlContext._"); } else { - intp.interpret("import sqlContext.implicits._"); - intp.interpret("import sqlContext.sql"); - intp.interpret("import org.apache.spark.sql.functions._"); + interpret("import sqlContext.implicits._"); + interpret("import sqlContext.sql"); + interpret("import org.apache.spark.sql.functions._"); } } } @@ -570,18 +606,20 @@ public void open() { Integer.parseInt(getProperty("zeppelin.spark.maxResult")) + ")"); */ - try { - if (sparkVersion.oldLoadFilesMethodName()) { - Method loadFiles = this.interpreter.getClass().getMethod("loadFiles", Settings.class); - loadFiles.invoke(this.interpreter, settings); - } else { - Method loadFiles = this.interpreter.getClass().getMethod( - "org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class); - loadFiles.invoke(this.interpreter, settings); + if (isScala2_10()) { + try { + if (sparkVersion.oldLoadFilesMethodName()) { + Method loadFiles = this.interpreter.getClass().getMethod("loadFiles", Settings.class); + loadFiles.invoke(this.interpreter, settings); + } else { + Method loadFiles = this.interpreter.getClass().getMethod( + "org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class); + loadFiles.invoke(this.interpreter, settings); + } + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + throw new InterpreterException(e); } - } catch (NoSuchMethodException | SecurityException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { - throw new InterpreterException(e); } // add jar from DepInterpreter @@ -625,6 +663,14 @@ public void open() { numReferenceOfSparkContext.incrementAndGet(); } + private Results.Result interpret(String line) { + return (Results.Result) invokeMethod( + intp, + "interpret", + new Class[] {String.class}, + new Object[] {line}); + } + private List currentClassPath() { List paths = classPath(Thread.currentThread().getContextClassLoader()); String[] cps = System.getProperty("java.class.path").split(File.pathSeparator); @@ -664,17 +710,22 @@ public List completion(String buf, int cursor) { completionText = ""; cursor = completionText.length(); } - ScalaCompleter c = completor.completer(); - Candidates ret = c.complete(completionText, cursor); + if (isScala2_10()) { + ScalaCompleter c = (ScalaCompleter) invokeMethod(completor, "completor"); + Candidates ret = c.complete(completionText, cursor); + + List candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates()); + List completions = new LinkedList(); - List candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates()); - List completions = new LinkedList(); + for (String candidate : candidates) { + completions.add(new InterpreterCompletion(candidate, candidate)); + } - for (String candidate : candidates) { - completions.add(new InterpreterCompletion(candidate, candidate)); + return completions; + } else { + return new LinkedList(); } - return completions; } private String getCompletionTargetString(String text, int cursor) { @@ -719,7 +770,7 @@ private String getCompletionTargetString(String text, int cursor) { } public Object getValue(String name) { - Object ret = intp.valueOfTerm(name); + Object ret = invokeMethod(intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); if (ret instanceof None) { return null; } else if (ret instanceof Some) { @@ -808,7 +859,7 @@ public InterpreterResult interpretInput(String[] lines, InterpreterContext conte scala.tools.nsc.interpreter.Results.Result res = null; try { - res = intp.interpret(incomplete + s); + res = interpret(incomplete + s); } catch (Exception e) { sc.clearJobGroup(); out.setInterpreterOutput(null); @@ -1000,7 +1051,7 @@ public void close() { sc = null; } - intp.close(); + invokeMethod(intp, "close"); } @Override @@ -1025,4 +1076,84 @@ public ZeppelinContext getZeppelinContext() { public SparkVersion getSparkVersion() { return sparkVersion; } + + private Object invokeMethod(Object o, String name) { + return invokeMethod(o, name, new Class[]{}, new Object[]{}); + } + + private Object invokeMethod(Object o, String name, Class [] argTypes, Object [] params) { + try { + return o.getClass().getMethod(name, argTypes).invoke(o, params); + } catch (NoSuchMethodException e) { + logger.error(e.getMessage(), e); + } catch (InvocationTargetException e) { + logger.error(e.getMessage(), e); + } catch (IllegalAccessException e) { + logger.error(e.getMessage(), e); + } + + return null; + } + + private Object invokeStaticMethod(Class c, String name) { + return invokeStaticMethod(c, name, new Class[]{}, new Object[]{}); + } + + private Object invokeStaticMethod(Class c, String name, Class [] argTypes, Object [] params) { + try { + return c.getMethod(name, argTypes).invoke(null, params); + } catch (NoSuchMethodException e) { + logger.error(e.getMessage(), e); + } catch (InvocationTargetException e) { + logger.error(e.getMessage(), e); + e.printStackTrace(); + } catch (IllegalAccessException e) { + logger.error(e.getMessage(), e); + } + + return null; + } + + private Object instantiateClass(String name, Class [] argTypes, Object [] params) { + try { + Constructor constructor = getClass().getClassLoader() + .loadClass(name).getConstructor(argTypes); + return constructor.newInstance(params); + } catch (NoSuchMethodException e) { + logger.error(e.getMessage(), e); + } catch (ClassNotFoundException e) { + logger.error(e.getMessage(), e); + } catch (IllegalAccessException e) { + logger.error(e.getMessage(), e); + } catch (InstantiationException e) { + logger.error(e.getMessage(), e); + } catch (InvocationTargetException e) { + logger.error(e.getMessage(), e); + } + return null; + } + + // function works after intp is initialized + private boolean isScala2_10() { + try { + this.getClass().forName("org.apache.spark.repl.SparkIMain"); + return true; + } catch (ClassNotFoundException e) { + return false; + } + } + + private boolean isScala2_11() { + return !isScala2_11(); + } + + + private Class findClass(String name) { + try { + return this.getClass().forName(name); + } catch (ClassNotFoundException e) { + logger.error(e.getMessage(), e); + return null; + } + } } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index a3636a29c1b..fc8923c4172 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -27,9 +27,7 @@ import org.apache.spark.sql.SQLContext; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java index 2fa716b449e..17f2de7be24 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java @@ -32,10 +32,12 @@ public class SparkVersion { public static final SparkVersion SPARK_1_4_0 = SparkVersion.fromVersionString("1.4.0"); public static final SparkVersion SPARK_1_5_0 = SparkVersion.fromVersionString("1.5.0"); public static final SparkVersion SPARK_1_6_0 = SparkVersion.fromVersionString("1.6.0"); - public static final SparkVersion SPARK_1_7_0 = SparkVersion.fromVersionString("1.7.0"); + + public static final SparkVersion SPARK_2_0_0 = SparkVersion.fromVersionString("2.0.0"); + public static final SparkVersion SPARK_2_1_0 = SparkVersion.fromVersionString("2.1.0"); public static final SparkVersion MIN_SUPPORTED_VERSION = SPARK_1_0_0; - public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_1_7_0; + public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_2_1_0; private int version; private String versionString; diff --git a/spark/src/main/scala-2.10/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java similarity index 94% rename from spark/src/main/scala-2.10/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java rename to spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java index e4881d373be..c4047977861 100644 --- a/spark/src/main/scala-2.10/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java @@ -29,7 +29,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.spark.SparkContext; -import org.apache.spark.repl.SparkIMain; import org.apache.zeppelin.dep.AbstractDependencyResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +59,7 @@ public class SparkDependencyResolver extends AbstractDependencyResolver { Logger logger = LoggerFactory.getLogger(SparkDependencyResolver.class); private Global global; - private SparkIMain intp; + private ClassLoader runtimeClassLoader; private SparkContext sc; private final String[] exclusions = new String[] {"org.scala-lang:scala-library", @@ -71,11 +70,14 @@ public class SparkDependencyResolver extends AbstractDependencyResolver { "org.apache.zeppelin:zeppelin-spark", "org.apache.zeppelin:zeppelin-server"}; - public SparkDependencyResolver(SparkIMain intp, SparkContext sc, String localRepoPath, - String additionalRemoteRepository) { + public SparkDependencyResolver(Global global, + ClassLoader runtimeClassLoader, + SparkContext sc, + String localRepoPath, + String additionalRemoteRepository) { super(localRepoPath); - this.intp = intp; - this.global = intp.global(); + this.global = global; + this.runtimeClassLoader = runtimeClassLoader; this.sc = sc; addRepoFromProperty(additionalRemoteRepository); } @@ -127,24 +129,22 @@ private void updateCompilerClassPath(URL[] urls) throws IllegalAccessException, private void updateRuntimeClassPath_1_x(URL[] urls) throws SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException { - ClassLoader cl = intp.classLoader().getParent(); Method addURL; - addURL = cl.getClass().getDeclaredMethod("addURL", new Class[] {URL.class}); + addURL = runtimeClassLoader.getClass().getDeclaredMethod("addURL", new Class[] {URL.class}); addURL.setAccessible(true); for (URL url : urls) { - addURL.invoke(cl, url); + addURL.invoke(runtimeClassLoader, url); } } private void updateRuntimeClassPath_2_x(URL[] urls) throws SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException { - ClassLoader cl = intp.classLoader().getParent(); Method addURL; - addURL = cl.getClass().getDeclaredMethod("addNewUrl", new Class[] {URL.class}); + addURL = runtimeClassLoader.getClass().getDeclaredMethod("addNewUrl", new Class[] {URL.class}); addURL.setAccessible(true); for (URL url : urls) { - addURL.invoke(cl, url); + addURL.invoke(runtimeClassLoader, url); } } @@ -209,7 +209,7 @@ public List load(String artifact, Collection excludes, private void loadFromFs(String artifact, boolean addSparkContext) throws Exception { File jarFile = new File(artifact); - intp.global().new Run(); + global.new Run(); if (sc.version().startsWith("1.1")) { updateRuntimeClassPath_1_x(new URL[] {jarFile.toURI().toURL()}); @@ -257,7 +257,7 @@ private List loadFromMvn(String artifact, Collection excludes, + artifactResult.getArtifact().getVersion()); } - intp.global().new Run(); + global.new Run(); if (sc.version().startsWith("1.1")) { updateRuntimeClassPath_1_x(newClassPathList.toArray(new URL[0])); } else { diff --git a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java deleted file mode 100644 index ee101789752..00000000000 --- a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/DepInterpreter.java +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark; - -import java.io.*; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.*; - -import org.apache.spark.repl.SparkILoop; -import org.apache.spark.repl.Main; -// import org.apache.spark.repl.SparkJLineCompletion; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.WrappedInterpreter; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.spark.dep.SparkDependencyContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.resolution.ArtifactResolutionException; -import org.sonatype.aether.resolution.DependencyResolutionException; - -import scala.Console; -import scala.None; -import scala.Some; -import scala.tools.nsc.Settings; -import scala.tools.nsc.interpreter.Completion.Candidates; -import scala.tools.nsc.interpreter.Completion.ScalaCompleter; -import scala.tools.nsc.settings.MutableSettings.BooleanSetting; -import scala.tools.nsc.settings.MutableSettings.PathSetting; - - -/** - * DepInterpreter downloads dependencies and pass them when SparkInterpreter initialized. - * It extends SparkInterpreter but does not create sparkcontext - * - */ -public class DepInterpreter extends Interpreter { - - static { - Interpreter.register( - "dep", - "spark", - DepInterpreter.class.getName(), - new InterpreterPropertyBuilder() - .add("zeppelin.dep.localrepo", - getSystemDefault("ZEPPELIN_DEP_LOCALREPO", null, "local-repo"), - "local repository for dependency loader") - .add("zeppelin.dep.additionalRemoteRepository", - "spark-packages,http://dl.bintray.com/spark-packages/maven,false;", - "A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository.") - .build()); - - } - - private Main intp; - private ByteArrayInputStream in; - private ByteArrayOutputStream out; - private SparkDependencyContext depc; - // private SparkJLineCompletion completor; - private SparkILoop interpreter; - static final Logger LOGGER = LoggerFactory.getLogger(DepInterpreter.class); - - public DepInterpreter(Properties property) { - super(property); - } - - public SparkDependencyContext getDependencyContext() { - return depc; - } - - public static String getSystemDefault( - String envName, - String propertyName, - String defaultValue) { - - if (envName != null && !envName.isEmpty()) { - String envValue = System.getenv().get(envName); - if (envValue != null) { - return envValue; - } - } - - if (propertyName != null && !propertyName.isEmpty()) { - String propValue = System.getProperty(propertyName); - if (propValue != null) { - return propValue; - } - } - return defaultValue; - } - - @Override - public void close() { - if (intp != null) { - intp.interp().interpreter().close(); - } - } - - @Override - public void open() { - out = new ByteArrayOutputStream(); - createIMain(); - } - - - private void createIMain() { - Settings settings = new Settings(); - URL[] urls = getClassloaderUrls(); - - // set classpath for scala compiler - PathSetting pathSettings = settings.classpath(); - String classpath = ""; - List paths = currentClassPath(); - for (File f : paths) { - if (classpath.length() > 0) { - classpath += File.pathSeparator; - } - classpath += f.getAbsolutePath(); - } - - if (urls != null) { - for (URL u : urls) { - if (classpath.length() > 0) { - classpath += File.pathSeparator; - } - classpath += u.getFile(); - } - } - - pathSettings.v_$eq(classpath); - settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings); - - // set classloader for scala compiler - settings.explicitParentLoader_$eq(new Some(Thread.currentThread() - .getContextClassLoader())); - - BooleanSetting b = (BooleanSetting) settings.usejavacp(); - b.v_$eq(true); - settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b); - - interpreter = new SparkILoop((java.io.BufferedReader) null, new PrintWriter(out)); - interpreter.settings_$eq(settings); - - interpreter.createInterpreter(); - - - intp.interp_$eq(this.interpreter); - intp.interp().interpreter().setContextClassLoader(); - intp.interp().interpreter().initializeSynchronous(); - - - depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"), - getProperty("zeppelin.dep.additionalRemoteRepository")); - // TODO(lresende) Support for completion - // completor = new SparkJLineCompletion(intp); - intp.interp().interpreter().interpret("" + - "@transient var _binder = new java.util.HashMap[String, Object]()"); - Map binder = (Map) getValue("_binder"); - binder.put("depc", depc); - - intp.interp().interpreter().interpret("@transient val z = " - + "_binder.get(\"depc\")" - + ".asInstanceOf[org.apache.zeppelin.spark.dep.SparkDependencyContext]"); - - } - - public Object getValue(String name) { - Object ret = intp.interp().interpreter().valueOfTerm(name); - if (ret instanceof None) { - return null; - } else if (ret instanceof Some) { - return ((Some) ret).get(); - } else { - return ret; - } - } - - @Override - public InterpreterResult interpret(String st, InterpreterContext context) { - PrintStream printStream = new PrintStream(out); - Console.setOut(printStream); - out.reset(); - - SparkInterpreter sparkInterpreter = getSparkInterpreter(); - - if (sparkInterpreter != null && sparkInterpreter.isSparkContextInitialized()) { - return new InterpreterResult(Code.ERROR, - "Must be used before SparkInterpreter (%spark) initialized\n" + - "Hint: put this paragraph before any Spark code and " + - "restart Zeppelin/Interpreter" ); - } - - scala.tools.nsc.interpreter.Results.Result ret = intp.interp().interpreter().interpret(st); - Code code = getResultCode(ret); - - try { - depc.fetch(); - } catch (MalformedURLException | DependencyResolutionException - | ArtifactResolutionException e) { - LOGGER.error("Exception in DepInterpreter while interpret ", e); - return new InterpreterResult(Code.ERROR, e.toString()); - } - - if (code == Code.INCOMPLETE) { - return new InterpreterResult(code, "Incomplete expression"); - } else if (code == Code.ERROR) { - return new InterpreterResult(code, out.toString()); - } else { - return new InterpreterResult(code, out.toString()); - } - } - - private Code getResultCode(scala.tools.nsc.interpreter.Results.Result r) { - if (r instanceof scala.tools.nsc.interpreter.Results.Success$) { - return Code.SUCCESS; - } else if (r instanceof scala.tools.nsc.interpreter.Results.Incomplete$) { - return Code.INCOMPLETE; - } else { - return Code.ERROR; - } - } - - @Override - public void cancel(InterpreterContext context) { - } - - - @Override - public FormType getFormType() { - return FormType.NATIVE; - } - - @Override - public int getProgress(InterpreterContext context) { - return 0; - } - - @Override - public List completion(String buf, int cursor) { - /* - ScalaCompleter c = completor.completer(); - Candidates ret = c.complete(buf, cursor); - return scala.collection.JavaConversions.seqAsJavaList(ret.candidates()); - */ - return Collections.emptyList(); - } - - private List currentClassPath() { - List paths = classPath(Thread.currentThread().getContextClassLoader()); - String[] cps = System.getProperty("java.class.path").split(File.pathSeparator); - if (cps != null) { - for (String cp : cps) { - paths.add(new File(cp)); - } - } - return paths; - } - - private List classPath(ClassLoader cl) { - List paths = new LinkedList(); - if (cl == null) { - return paths; - } - - if (cl instanceof URLClassLoader) { - URLClassLoader ucl = (URLClassLoader) cl; - URL[] urls = ucl.getURLs(); - if (urls != null) { - for (URL url : urls) { - paths.add(new File(url.getFile())); - } - } - } - return paths; - } - - private SparkInterpreter getSparkInterpreter() { - InterpreterGroup intpGroup = getInterpreterGroup(); - if (intpGroup == null) { - return null; - } - - Interpreter p = getInterpreterInTheSameSessionByClassName(SparkInterpreter.class.getName()); - if (p == null) { - return null; - } - - while (p instanceof WrappedInterpreter) { - p = ((WrappedInterpreter) p).getInnerInterpreter(); - } - return (SparkInterpreter) p; - } - - @Override - public Scheduler getScheduler() { - SparkInterpreter sparkInterpreter = getSparkInterpreter(); - if (sparkInterpreter != null) { - return getSparkInterpreter().getScheduler(); - } else { - return null; - } - } -} diff --git a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java deleted file mode 100644 index 7c694232c9d..00000000000 --- a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/SparkInterpreter.java +++ /dev/null @@ -1,1006 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark; - -import java.io.File; -import java.io.PrintWriter; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; - -import com.google.common.base.Joiner; - -import org.apache.spark.HttpServer; -import org.apache.spark.SparkConf; -import org.apache.spark.SparkContext; -import org.apache.spark.SparkEnv; -// import org.apache.spark.repl.SparkCommandLine; -import org.apache.spark.repl.SparkILoop; -import org.apache.spark.repl.Main; -// import org.apache.spark.repl.SparkJLineCompletion; -import org.apache.spark.scheduler.ActiveJob; -import org.apache.spark.scheduler.DAGScheduler; -import org.apache.spark.scheduler.Pool; -import org.apache.spark.sql.SQLContext; -import org.apache.spark.ui.jobs.JobProgressListener; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.InterpreterUtils; -import org.apache.zeppelin.interpreter.WrappedInterpreter; -import org.apache.zeppelin.scheduler.Scheduler; -import org.apache.zeppelin.scheduler.SchedulerFactory; -import org.apache.zeppelin.spark.dep.SparkDependencyContext; -import org.apache.zeppelin.spark.dep.SparkDependencyResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import scala.*; -import scala.Enumeration.Value; -import scala.collection.Iterator; -import scala.collection.JavaConversions; -import scala.collection.JavaConverters; -import scala.collection.Seq; -import scala.collection.mutable.HashMap; -import scala.collection.mutable.HashSet; -import scala.reflect.io.AbstractFile; -import scala.tools.nsc.Settings; -import scala.tools.nsc.interpreter.Completion.Candidates; -import scala.tools.nsc.interpreter.Completion.ScalaCompleter; -import scala.tools.nsc.settings.MutableSettings.BooleanSetting; -import scala.tools.nsc.settings.MutableSettings.PathSetting; - -/** - * Spark interpreter for Zeppelin. - * - */ -public class SparkInterpreter extends Interpreter { - public static Logger logger = LoggerFactory.getLogger(SparkInterpreter.class); - - static { - Interpreter.register( - "spark", - "spark", - SparkInterpreter.class.getName(), - new InterpreterPropertyBuilder() - .add("spark.app.name", - getSystemDefault("SPARK_APP_NAME", "spark.app.name", "Zeppelin"), - "The name of spark application.") - .add("master", - getSystemDefault("MASTER", "spark.master", "local[*]"), - "Spark master uri. ex) spark://masterhost:7077") - .add("spark.executor.memory", - getSystemDefault(null, "spark.executor.memory", ""), - "Executor memory per worker instance. ex) 512m, 32g") - .add("spark.cores.max", - getSystemDefault(null, "spark.cores.max", ""), - "Total number of cores to use. Empty value uses all available core.") - .add("zeppelin.spark.useHiveContext", - getSystemDefault("ZEPPELIN_SPARK_USEHIVECONTEXT", - "zeppelin.spark.useHiveContext", "true"), - "Use HiveContext instead of SQLContext if it is true.") - .add("zeppelin.spark.maxResult", - getSystemDefault("ZEPPELIN_SPARK_MAXRESULT", "zeppelin.spark.maxResult", "1000"), - "Max number of SparkSQL result to display.") - .add("args", "", "spark commandline args") - .add("zeppelin.spark.printREPLOutput", "true", - "Print REPL output") - .build() - ); - } - - private ZeppelinContext z; - private SparkILoop interpreter; - private Main intp; - private static SparkContext sc; - private static SQLContext sqlc; - private static SparkEnv env; - private static JobProgressListener sparkListener; - private static AbstractFile classOutputDir; - private static Integer sharedInterpreterLock = new Integer(0); - private static AtomicInteger numReferenceOfSparkContext = new AtomicInteger(0); - - private SparkOutputStream out; - private SparkDependencyResolver dep; - // private SparkJLineCompletion completor; - - private Map binder; - private SparkVersion sparkVersion; - - - public SparkInterpreter(Properties property) { - super(property); - out = new SparkOutputStream(); - } - - public SparkInterpreter(Properties property, SparkContext sc) { - this(property); - - this.sc = sc; - env = SparkEnv.get(); - sparkListener = setupListeners(this.sc); - } - - public SparkContext getSparkContext() { - synchronized (sharedInterpreterLock) { - if (sc == null) { - sc = createSparkContext(); - env = SparkEnv.get(); - sparkListener = setupListeners(sc); - } - return sc; - } - } - - public boolean isSparkContextInitialized() { - synchronized (sharedInterpreterLock) { - return sc != null; - } - } - - static JobProgressListener setupListeners(SparkContext context) { - JobProgressListener pl = new JobProgressListener(context.getConf()); - try { - Object listenerBus = context.getClass().getMethod("listenerBus").invoke(context); - - Method[] methods = listenerBus.getClass().getMethods(); - Method addListenerMethod = null; - for (Method m : methods) { - if (!m.getName().equals("addListener")) { - continue; - } - - Class[] parameterTypes = m.getParameterTypes(); - - if (parameterTypes.length != 1) { - continue; - } - - if (!parameterTypes[0].isAssignableFrom(JobProgressListener.class)) { - continue; - } - - addListenerMethod = m; - break; - } - - if (addListenerMethod != null) { - addListenerMethod.invoke(listenerBus, pl); - } else { - return null; - } - } catch (NoSuchMethodException | SecurityException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { - logger.error(e.toString(), e); - return null; - } - return pl; - } - - private boolean useHiveContext() { - return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.useHiveContext")); - } - - public SQLContext getSQLContext() { - synchronized (sharedInterpreterLock) { - if (sqlc == null) { - if (useHiveContext()) { - String name = "org.apache.spark.sql.hive.HiveContext"; - Constructor hc; - try { - hc = getClass().getClassLoader().loadClass(name) - .getConstructor(SparkContext.class); - sqlc = (SQLContext) hc.newInstance(getSparkContext()); - } catch (NoSuchMethodException | SecurityException - | ClassNotFoundException | InstantiationException - | IllegalAccessException | IllegalArgumentException - | InvocationTargetException e) { - logger.warn("Can't create HiveContext. Fallback to SQLContext", e); - // when hive dependency is not loaded, it'll fail. - // in this case SQLContext can be used. - sqlc = new SQLContext(getSparkContext()); - } - } else { - sqlc = new SQLContext(getSparkContext()); - } - } - return sqlc; - } - } - - public SparkDependencyResolver getDependencyResolver() { - if (dep == null) { - dep = new SparkDependencyResolver(intp, - sc, - getProperty("zeppelin.dep.localrepo"), - getProperty("zeppelin.dep.additionalRemoteRepository")); - } - return dep; - } - - private DepInterpreter getDepInterpreter() { - Interpreter p = getInterpreterInTheSameSessionByClassName(DepInterpreter.class.getName()); - if (p == null) { - return null; - } - - while (p instanceof WrappedInterpreter) { - p = ((WrappedInterpreter) p).getInnerInterpreter(); - } - return (DepInterpreter) p; - } - - public SparkContext createSparkContext() { - System.err.println("------ Create new SparkContext " + getProperty("master") + " -------"); - - String execUri = System.getenv("SPARK_EXECUTOR_URI"); - // TODO(lresende) Figure out added jars ==>> SparkILoop.getAddedJars(); - String[] jars = new String[0]; - - String classServerUri = null; - - try { // in case of spark 1.1x, spark 1.2x - Method classServer = interpreter.intp().getClass().getMethod("classServer"); - HttpServer httpServer = (HttpServer) classServer.invoke(interpreter.intp()); - classServerUri = httpServer.uri(); - } catch (NoSuchMethodException | SecurityException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { - // continue - } - - if (classServerUri == null) { - try { // for spark 1.3x - Method classServer = interpreter.intp().getClass().getMethod("classServerUri"); - classServerUri = (String) classServer.invoke(interpreter.intp()); - } catch (NoSuchMethodException | SecurityException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { - throw new InterpreterException(e); - } - } - - SparkConf conf = - new SparkConf() - .setMaster(getProperty("master")) - .setAppName(getProperty("spark.app.name")) - .set("spark.repl.class.uri", classServerUri); - - if (jars.length > 0) { - conf.setJars(jars); - } - - if (execUri != null) { - conf.set("spark.executor.uri", execUri); - } - if (System.getenv("SPARK_HOME") != null) { - conf.setSparkHome(System.getenv("SPARK_HOME")); - } - conf.set("spark.scheduler.mode", "FAIR"); - - Properties intpProperty = getProperty(); - - for (Object k : intpProperty.keySet()) { - String key = (String) k; - String val = toString(intpProperty.get(key)); - if (!key.startsWith("spark.") || !val.trim().isEmpty()) { - logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val)); - conf.set(key, val); - } - } - - //TODO(jongyoul): Move these codes into PySparkInterpreter.java - String pysparkBasePath = getSystemDefault("SPARK_HOME", null, null); - File pysparkPath; - if (null == pysparkBasePath) { - pysparkBasePath = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../"); - pysparkPath = new File(pysparkBasePath, - "interpreter" + File.separator + "spark" + File.separator + "pyspark"); - } else { - pysparkPath = new File(pysparkBasePath, - "python" + File.separator + "lib"); - } - - //Only one of py4j-0.9-src.zip and py4j-0.8.2.1-src.zip should exist - String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip"}; - ArrayList pythonLibUris = new ArrayList<>(); - for (String lib : pythonLibs) { - File libFile = new File(pysparkPath, lib); - if (libFile.exists()) { - pythonLibUris.add(libFile.toURI().toString()); - } - } - pythonLibUris.trimToSize(); - - // Distribute two libraries(pyspark.zip and py4j-*.zip) to workers - // when spark version is less than or equal to 1.4.1 - if (pythonLibUris.size() == 2) { - try { - String confValue = conf.get("spark.yarn.dist.files"); - conf.set("spark.yarn.dist.files", confValue + "," + Joiner.on(",").join(pythonLibUris)); - } catch (NoSuchElementException e) { - conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris)); - } - if (!useSparkSubmit()) { - conf.set("spark.files", conf.get("spark.yarn.dist.files")); - } - conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs)); - } - - // Distributes needed libraries to workers - // when spark version is greater than or equal to 1.5.0 - if (getProperty("master").equals("yarn-client")) { - conf.set("spark.yarn.isPython", "true"); - } - - SparkContext sparkContext = new SparkContext(conf); - return sparkContext; - } - - static final String toString(Object o) { - return (o instanceof String) ? (String) o : ""; - } - - private boolean useSparkSubmit() { - return null != System.getenv("SPARK_SUBMIT"); - } - - public static String getSystemDefault( - String envName, - String propertyName, - String defaultValue) { - - if (envName != null && !envName.isEmpty()) { - String envValue = System.getenv().get(envName); - if (envValue != null) { - return envValue; - } - } - - if (propertyName != null && !propertyName.isEmpty()) { - String propValue = System.getProperty(propertyName); - if (propValue != null) { - return propValue; - } - } - return defaultValue; - } - - public boolean printREPLOutput() { - return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.printREPLOutput")); - } - - @Override - public void open() { - URL[] urls = getClassloaderUrls(); - - // Very nice discussion about how scala compiler handle classpath - // https://groups.google.com/forum/#!topic/scala-user/MlVwo2xCCI0 - - Settings settings = new Settings(); - if (getProperty("args") != null) { - String[] argsArray = getProperty("args").split(" "); - LinkedList argList = new LinkedList(); - for (String arg : argsArray) { - argList.add(arg); - } - - /* - SparkCommandLine command = - new SparkCommandLine(scala.collection.JavaConversions.asScalaBuffer(argList).toList()); - settings = command.settings(); - */ - } - - // set classpath for scala compiler - PathSetting pathSettings = settings.classpath(); - String classpath = ""; - List paths = currentClassPath(); - for (File f : paths) { - if (classpath.length() > 0) { - classpath += File.pathSeparator; - } - classpath += f.getAbsolutePath(); - } - - if (urls != null) { - for (URL u : urls) { - if (classpath.length() > 0) { - classpath += File.pathSeparator; - } - classpath += u.getFile(); - } - } - - // add dependency from DepInterpreter - DepInterpreter depInterpreter = getDepInterpreter(); - if (depInterpreter != null) { - SparkDependencyContext depc = depInterpreter.getDependencyContext(); - if (depc != null) { - List files = depc.getFiles(); - if (files != null) { - for (File f : files) { - if (classpath.length() > 0) { - classpath += File.pathSeparator; - } - classpath += f.getAbsolutePath(); - } - } - } - } - - // add dependency from local repo - String localRepo = getProperty("zeppelin.interpreter.localRepo"); - if (localRepo != null) { - File localRepoDir = new File(localRepo); - if (localRepoDir.exists()) { - File[] files = localRepoDir.listFiles(); - if (files != null) { - for (File f : files) { - if (classpath.length() > 0) { - classpath += File.pathSeparator; - } - classpath += f.getAbsolutePath(); - } - } - } - } - - pathSettings.v_$eq(classpath); - settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings); - - - // set classloader for scala compiler - settings.explicitParentLoader_$eq( - new Some(Thread.currentThread().getContextClassLoader())); - BooleanSetting b = (BooleanSetting) settings.usejavacp(); - b.v_$eq(true); - settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b); - - System.setProperty("scala.repl.name.line", "line" + this.hashCode() + "$"); - - synchronized (sharedInterpreterLock) { - /* create scala repl */ - if (printREPLOutput()) { - this.interpreter = new SparkILoop((java.io.BufferedReader) null, - new PrintWriter(out)); - } else { - this.interpreter = new SparkILoop((java.io.BufferedReader) null, - new PrintWriter(Console.out())); - } - - interpreter.settings_$eq(settings); - - interpreter.createInterpreter(); - - intp.interp_$eq(this.interpreter); - intp.interp().interpreter().setContextClassLoader(); - intp.interp().interpreter().initializeSynchronous(); - - if (classOutputDir == null) { - classOutputDir = settings.outputDirs().getSingleOutput().get(); - } else { - // change Main class output dir - settings.outputDirs().setSingleOutput(classOutputDir); - ClassLoader cl = intp.interp().interpreter().classLoader(); - - try { - Field rootField = cl.getClass().getSuperclass().getDeclaredField("root"); - rootField.setAccessible(true); - rootField.set(cl, classOutputDir); - } catch (NoSuchFieldException | IllegalAccessException e) { - logger.error(e.getMessage(), e); - } - } - - // TODO(lresende) support for completion - // completor = new SparkJLineCompletion(intp); - - sc = getSparkContext(); - if (sc.getPoolForName("fair").isEmpty()) { - Value schedulingMode = org.apache.spark.scheduler.SchedulingMode.FAIR(); - int minimumShare = 0; - int weight = 1; - Pool pool = new Pool("fair", schedulingMode, minimumShare, weight); - sc.taskScheduler().rootPool().addSchedulable(pool); - } - - sparkVersion = SparkVersion.fromVersionString(sc.version()); - - sqlc = getSQLContext(); - - dep = getDependencyResolver(); - - z = new ZeppelinContext(sc, sqlc, null, dep, - Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); - - intp.interp().interpreter().interpret("" + - "@transient var _binder = new java.util.HashMap[String, Object]()"); - binder = (Map) getValue("_binder"); - binder.put("sc", sc); - binder.put("sqlc", sqlc); - binder.put("z", z); - - intp.interp().interpreter().interpret("@transient val z = " - + "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]"); - intp.interp().interpreter().interpret("@transient val sc = " - + "_binder.get(\"sc\").asInstanceOf[org.apache.spark.SparkContext]"); - intp.interp().interpreter().interpret("@transient val sqlc = " - + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); - intp.interp().interpreter().interpret("@transient val sqlContext = " - + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); - intp.interp().interpreter().interpret("import org.apache.spark.SparkContext._"); - - if (sparkVersion.oldSqlContextImplicits()) { - intp.interp().interpreter().interpret("import sqlContext._"); - } else { - intp.interp().interpreter().interpret("import sqlContext.implicits._"); - intp.interp().interpreter().interpret("import sqlContext.sql"); - intp.interp().interpreter().interpret("import org.apache.spark.sql.functions._"); - } - } - - /* Temporary disabling DisplayUtils. see https://issues.apache.org/jira/browse/ZEPPELIN-127 - * - // Utility functions for display - intp.interpret("import org.apache.zeppelin.spark.utils.DisplayUtils._"); - - // Scala implicit value for spark.maxResult - intp.interpret("import org.apache.zeppelin.spark.utils.SparkMaxResult"); - intp.interpret("implicit val sparkMaxResult = new SparkMaxResult(" + - Integer.parseInt(getProperty("zeppelin.spark.maxResult")) + ")"); - */ - - try { - if (sparkVersion.oldLoadFilesMethodName()) { - Method loadFiles = this.interpreter.getClass().getMethod("loadFiles", Settings.class); - loadFiles.invoke(this.interpreter, settings); - } else { - Method loadFiles = this.interpreter.getClass().getMethod( - "org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class); - loadFiles.invoke(this.interpreter, settings); - } - } catch (NoSuchMethodException | SecurityException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException e) { - throw new InterpreterException(e); - } - - // add jar from DepInterpreter - if (depInterpreter != null) { - SparkDependencyContext depc = depInterpreter.getDependencyContext(); - if (depc != null) { - List files = depc.getFilesDist(); - if (files != null) { - for (File f : files) { - if (f.getName().toLowerCase().endsWith(".jar")) { - sc.addJar(f.getAbsolutePath()); - logger.info("sc.addJar(" + f.getAbsolutePath() + ")"); - } else { - sc.addFile(f.getAbsolutePath()); - logger.info("sc.addFile(" + f.getAbsolutePath() + ")"); - } - } - } - } - } - - // add jar from local repo - if (localRepo != null) { - File localRepoDir = new File(localRepo); - if (localRepoDir.exists()) { - File[] files = localRepoDir.listFiles(); - if (files != null) { - for (File f : files) { - if (f.getName().toLowerCase().endsWith(".jar")) { - sc.addJar(f.getAbsolutePath()); - logger.info("sc.addJar(" + f.getAbsolutePath() + ")"); - } else { - sc.addFile(f.getAbsolutePath()); - logger.info("sc.addFile(" + f.getAbsolutePath() + ")"); - } - } - } - } - } - - numReferenceOfSparkContext.incrementAndGet(); - } - - private List currentClassPath() { - List paths = classPath(Thread.currentThread().getContextClassLoader()); - String[] cps = System.getProperty("java.class.path").split(File.pathSeparator); - if (cps != null) { - for (String cp : cps) { - paths.add(new File(cp)); - } - } - return paths; - } - - private List classPath(ClassLoader cl) { - List paths = new LinkedList(); - if (cl == null) { - return paths; - } - - if (cl instanceof URLClassLoader) { - URLClassLoader ucl = (URLClassLoader) cl; - URL[] urls = ucl.getURLs(); - if (urls != null) { - for (URL url : urls) { - paths.add(new File(url.getFile())); - } - } - } - return paths; - } - - @Override - public List completion(String buf, int cursor) { - /* - if (buf.length() < cursor) { - cursor = buf.length(); - } - String completionText = getCompletionTargetString(buf, cursor); - if (completionText == null) { - completionText = ""; - cursor = completionText.length(); - } - ScalaCompleter c = completor.completer(); - Candidates ret = c.complete(completionText, cursor); - return scala.collection.JavaConversions.seqAsJavaList(ret.candidates()); - */ - return Collections.emptyList(); - } - - private String getCompletionTargetString(String text, int cursor) { - String[] completionSeqCharaters = {" ", "\n", "\t"}; - int completionEndPosition = cursor; - int completionStartPosition = cursor; - int indexOfReverseSeqPostion = cursor; - - String resultCompletionText = ""; - String completionScriptText = ""; - try { - completionScriptText = text.substring(0, cursor); - } - catch (Exception e) { - logger.error(e.toString()); - return null; - } - completionEndPosition = completionScriptText.length(); - - String tempReverseCompletionText = new StringBuilder(completionScriptText).reverse().toString(); - - for (String seqCharacter : completionSeqCharaters) { - indexOfReverseSeqPostion = tempReverseCompletionText.indexOf(seqCharacter); - - if (indexOfReverseSeqPostion < completionStartPosition && indexOfReverseSeqPostion > 0) { - completionStartPosition = indexOfReverseSeqPostion; - } - - } - - if (completionStartPosition == completionEndPosition) { - completionStartPosition = 0; - } - else - { - completionStartPosition = completionEndPosition - completionStartPosition; - } - resultCompletionText = completionScriptText.substring( - completionStartPosition , completionEndPosition); - - return resultCompletionText; - } - - public Object getValue(String name) { - Object ret = intp.interp().interpreter().valueOfTerm(name); - if (ret instanceof None) { - return null; - } else if (ret instanceof Some) { - return ((Some) ret).get(); - } else { - return ret; - } - } - - String getJobGroup(InterpreterContext context){ - return "zeppelin-" + context.getParagraphId(); - } - - /** - * Interpret a single line. - */ - @Override - public InterpreterResult interpret(String line, InterpreterContext context) { - if (sparkVersion.isUnsupportedVersion()) { - return new InterpreterResult(Code.ERROR, "Spark " + sparkVersion.toString() - + " is not supported"); - } - - z.setInterpreterContext(context); - if (line == null || line.trim().length() == 0) { - return new InterpreterResult(Code.SUCCESS); - } - return interpret(line.split("\n"), context); - } - - public InterpreterResult interpret(String[] lines, InterpreterContext context) { - synchronized (this) { - z.setGui(context.getGui()); - sc.setJobGroup(getJobGroup(context), "Zeppelin", false); - InterpreterResult r = interpretInput(lines, context); - sc.clearJobGroup(); - return r; - } - } - - public InterpreterResult interpretInput(String[] lines, InterpreterContext context) { - SparkEnv.set(env); - - // add print("") to make sure not finishing with comment - // see https://github.com/NFLabs/zeppelin/issues/151 - String[] linesToRun = new String[lines.length + 1]; - for (int i = 0; i < lines.length; i++) { - linesToRun[i] = lines[i]; - } - linesToRun[lines.length] = "print(\"\")"; - - Console.setOut(context.out); - out.setInterpreterOutput(context.out); - context.out.clear(); - Code r = null; - String incomplete = ""; - boolean inComment = false; - - for (int l = 0; l < linesToRun.length; l++) { - String s = linesToRun[l]; - // check if next line starts with "." (but not ".." or "./") it is treated as an invocation - if (l + 1 < linesToRun.length) { - String nextLine = linesToRun[l + 1].trim(); - boolean continuation = false; - if (nextLine.isEmpty() - || nextLine.startsWith("//") // skip empty line or comment - || nextLine.startsWith("}") - || nextLine.startsWith("object")) { // include "} object" for Scala companion object - continuation = true; - } else if (!inComment && nextLine.startsWith("/*")) { - inComment = true; - continuation = true; - } else if (inComment && nextLine.lastIndexOf("*/") >= 0) { - inComment = false; - continuation = true; - } else if (nextLine.length() > 1 - && nextLine.charAt(0) == '.' - && nextLine.charAt(1) != '.' // ".." - && nextLine.charAt(1) != '/') { // "./" - continuation = true; - } else if (inComment) { - continuation = true; - } - if (continuation) { - incomplete += s + "\n"; - continue; - } - } - - scala.tools.nsc.interpreter.Results.Result res = null; - try { - res = intp.interp().interpreter().interpret(incomplete + s); - } catch (Exception e) { - sc.clearJobGroup(); - out.setInterpreterOutput(null); - logger.info("Interpreter exception", e); - return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e)); - } - - r = getResultCode(res); - - if (r == Code.ERROR) { - sc.clearJobGroup(); - out.setInterpreterOutput(null); - return new InterpreterResult(r, ""); - } else if (r == Code.INCOMPLETE) { - incomplete += s + "\n"; - } else { - incomplete = ""; - } - } - - if (r == Code.INCOMPLETE) { - sc.clearJobGroup(); - out.setInterpreterOutput(null); - return new InterpreterResult(r, "Incomplete expression"); - } else { - sc.clearJobGroup(); - out.setInterpreterOutput(null); - return new InterpreterResult(Code.SUCCESS); - } - } - - - @Override - public void cancel(InterpreterContext context) { - sc.cancelJobGroup(getJobGroup(context)); - } - - @Override - public int getProgress(InterpreterContext context) { - String jobGroup = getJobGroup(context); - int completedTasks = 0; - int totalTasks = 0; - - DAGScheduler scheduler = sc.dagScheduler(); - if (scheduler == null) { - return 0; - } - HashSet jobs = scheduler.activeJobs(); - if (jobs == null || jobs.size() == 0) { - return 0; - } - Iterator it = jobs.iterator(); - while (it.hasNext()) { - ActiveJob job = it.next(); - String g = (String) job.properties().get("spark.jobGroup.id"); - if (jobGroup.equals(g)) { - int[] progressInfo = null; - try { - Object finalStage = job.getClass().getMethod("finalStage").invoke(job); - if (sparkVersion.getProgress1_0()) { - progressInfo = getProgressFromStage_1_0x(sparkListener, finalStage); - } else { - progressInfo = getProgressFromStage_1_1x(sparkListener, finalStage); - } - } catch (IllegalAccessException | IllegalArgumentException - | InvocationTargetException | NoSuchMethodException - | SecurityException e) { - logger.error("Can't get progress info", e); - return 0; - } - totalTasks += progressInfo[0]; - completedTasks += progressInfo[1]; - } - } - - if (totalTasks == 0) { - return 0; - } - return completedTasks * 100 / totalTasks; - } - - private int[] getProgressFromStage_1_0x(JobProgressListener sparkListener, Object stage) - throws IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException { - int numTasks = (int) stage.getClass().getMethod("numTasks").invoke(stage); - int completedTasks = 0; - - int id = (int) stage.getClass().getMethod("id").invoke(stage); - - Object completedTaskInfo = null; - - completedTaskInfo = JavaConversions.mapAsJavaMap( - (HashMap) sparkListener.getClass() - .getMethod("stageIdToTasksComplete").invoke(sparkListener)).get(id); - - if (completedTaskInfo != null) { - completedTasks += (int) completedTaskInfo; - } - List parents = JavaConversions.seqAsJavaList((Seq) stage.getClass() - .getMethod("parents").invoke(stage)); - if (parents != null) { - for (Object s : parents) { - int[] p = getProgressFromStage_1_0x(sparkListener, s); - numTasks += p[0]; - completedTasks += p[1]; - } - } - - return new int[] {numTasks, completedTasks}; - } - - private int[] getProgressFromStage_1_1x(JobProgressListener sparkListener, Object stage) - throws IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException { - int numTasks = (int) stage.getClass().getMethod("numTasks").invoke(stage); - int completedTasks = 0; - int id = (int) stage.getClass().getMethod("id").invoke(stage); - - try { - Method stageIdToData = sparkListener.getClass().getMethod("stageIdToData"); - HashMap, Object> stageIdData = - (HashMap, Object>) stageIdToData.invoke(sparkListener); - Class stageUIDataClass = - this.getClass().forName("org.apache.spark.ui.jobs.UIData$StageUIData"); - - Method numCompletedTasks = stageUIDataClass.getMethod("numCompleteTasks"); - Set> keys = - JavaConverters.setAsJavaSetConverter(stageIdData.keySet()).asJava(); - for (Tuple2 k : keys) { - if (id == (int) k._1()) { - Object uiData = stageIdData.get(k).get(); - completedTasks += (int) numCompletedTasks.invoke(uiData); - } - } - } catch (Exception e) { - logger.error("Error on getting progress information", e); - } - - List parents = JavaConversions.seqAsJavaList((Seq) stage.getClass() - .getMethod("parents").invoke(stage)); - if (parents != null) { - for (Object s : parents) { - int[] p = getProgressFromStage_1_1x(sparkListener, s); - numTasks += p[0]; - completedTasks += p[1]; - } - } - return new int[] {numTasks, completedTasks}; - } - - private Code getResultCode(scala.tools.nsc.interpreter.Results.Result r) { - if (r instanceof scala.tools.nsc.interpreter.Results.Success$) { - return Code.SUCCESS; - } else if (r instanceof scala.tools.nsc.interpreter.Results.Incomplete$) { - return Code.INCOMPLETE; - } else { - return Code.ERROR; - } - } - - @Override - public void close() { - logger.info("Close interpreter"); - - if (numReferenceOfSparkContext.decrementAndGet() == 0) { - sc.stop(); - sc = null; - } - - intp.interp().interpreter().close(); - } - - @Override - public FormType getFormType() { - return FormType.NATIVE; - } - - public JobProgressListener getJobProgressListener() { - return sparkListener; - } - - @Override - public Scheduler getScheduler() { - return SchedulerFactory.singleton().createOrGetFIFOScheduler( - SparkInterpreter.class.getName() + this.hashCode()); - } - - public ZeppelinContext getZeppelinContext() { - return z; - } - - public SparkVersion getSparkVersion() { - return sparkVersion; - } -} diff --git a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java b/spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java deleted file mode 100644 index 9c00c556cef..00000000000 --- a/spark/src/main/scala-2.11/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java +++ /dev/null @@ -1,351 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.spark.dep; - -import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.net.URL; -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.spark.SparkContext; -import org.apache.spark.repl.Main; -import org.apache.zeppelin.dep.AbstractDependencyResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.artifact.Artifact; -import org.sonatype.aether.collection.CollectRequest; -import org.sonatype.aether.graph.Dependency; -import org.sonatype.aether.graph.DependencyFilter; -import org.sonatype.aether.repository.RemoteRepository; -import org.sonatype.aether.resolution.ArtifactResult; -import org.sonatype.aether.resolution.DependencyRequest; -import org.sonatype.aether.util.artifact.DefaultArtifact; -import org.sonatype.aether.util.artifact.JavaScopes; -import org.sonatype.aether.util.filter.DependencyFilterUtils; -import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter; - -import scala.Some; -import scala.collection.IndexedSeq; -import scala.reflect.io.AbstractFile; -import scala.tools.nsc.Global; -import scala.tools.nsc.backend.JavaPlatform; -import scala.tools.nsc.util.ClassPath; -import scala.tools.nsc.util.MergedClassPath; - -/** - * Deps resolver. - * Add new dependencies from mvn repo (at runtime) to Spark interpreter group. - */ -public class SparkDependencyResolver extends AbstractDependencyResolver { - Logger logger = LoggerFactory.getLogger(SparkDependencyResolver.class); - private Global global; - private Main intp; - private SparkContext sc; - - private final String[] exclusions = new String[] {"org.scala-lang:scala-library", - "org.scala-lang:scala-compiler", - "org.scala-lang:scala-reflect", - "org.scala-lang:scalap", - "org.apache.zeppelin:zeppelin-zengine", - "org.apache.zeppelin:zeppelin-spark", - "org.apache.zeppelin:zeppelin-server"}; - - public SparkDependencyResolver(Main intp, SparkContext sc, String localRepoPath, - String additionalRemoteRepository) { - super(localRepoPath); - this.intp = intp; - this.global = intp.interp().interpreter().global(); - this.sc = sc; - addRepoFromProperty(additionalRemoteRepository); - } - - private void addRepoFromProperty(String listOfRepo) { - if (listOfRepo != null) { - String[] repos = listOfRepo.split(";"); - for (String repo : repos) { - String[] parts = repo.split(","); - if (parts.length == 3) { - String id = parts[0].trim(); - String url = parts[1].trim(); - boolean isSnapshot = Boolean.parseBoolean(parts[2].trim()); - if (id.length() > 1 && url.length() > 1) { - addRepo(id, url, isSnapshot); - } - } - } - } - } - - private void updateCompilerClassPath(URL[] urls) throws IllegalAccessException, - IllegalArgumentException, InvocationTargetException { - - JavaPlatform platform = (JavaPlatform) global.platform(); - MergedClassPath newClassPath = mergeUrlsIntoClassPath(platform, urls); - - Method[] methods = platform.getClass().getMethods(); - for (Method m : methods) { - if (m.getName().endsWith("currentClassPath_$eq")) { - m.invoke(platform, new Some(newClassPath)); - break; - } - } - - // NOTE: Must use reflection until this is exposed/fixed upstream in Scala - List classPaths = new LinkedList(); - for (URL url : urls) { - classPaths.add(url.getPath()); - } - - // Reload all jars specified into our compiler - global.invalidateClassPathEntries(scala.collection.JavaConversions.asScalaBuffer(classPaths) - .toList()); - } - - // Until spark 1.1.x - // check https://github.com/apache/spark/commit/191d7cf2a655d032f160b9fa181730364681d0e7 - private void updateRuntimeClassPath_1_x(URL[] urls) throws SecurityException, - IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException { - ClassLoader cl = intp.interp().interpreter().classLoader().getParent(); - Method addURL; - addURL = cl.getClass().getDeclaredMethod("addURL", new Class[] {URL.class}); - addURL.setAccessible(true); - for (URL url : urls) { - addURL.invoke(cl, url); - } - } - - private void updateRuntimeClassPath_2_x(URL[] urls) throws SecurityException, - IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException { - ClassLoader cl = intp.interp().interpreter().classLoader().getParent(); - Method addURL; - addURL = cl.getClass().getDeclaredMethod("addNewUrl", new Class[] {URL.class}); - addURL.setAccessible(true); - for (URL url : urls) { - addURL.invoke(cl, url); - } - } - - private MergedClassPath mergeUrlsIntoClassPath(JavaPlatform platform, URL[] urls) { - IndexedSeq> entries = - ((MergedClassPath) platform.classPath()).entries(); - List> cp = new LinkedList>(); - - for (int i = 0; i < entries.size(); i++) { - cp.add(entries.apply(i)); - } - - for (URL url : urls) { - AbstractFile file; - if ("file".equals(url.getProtocol())) { - File f = new File(url.getPath()); - if (f.isDirectory()) { - file = AbstractFile.getDirectory(scala.reflect.io.File.jfile2path(f)); - } else { - file = AbstractFile.getFile(scala.reflect.io.File.jfile2path(f)); - } - } else { - file = AbstractFile.getURL(url); - } - - ClassPath newcp = platform.classPath().context().newClassPath(file); - - // distinct - if (cp.contains(newcp) == false) { - cp.add(newcp); - } - } - - return new MergedClassPath(scala.collection.JavaConversions.asScalaBuffer(cp).toIndexedSeq(), - platform.classPath().context()); - } - - public List load(String artifact, - boolean addSparkContext) throws Exception { - return load(artifact, new LinkedList(), addSparkContext); - } - - public List load(String artifact, Collection excludes, - boolean addSparkContext) throws Exception { - if (StringUtils.isBlank(artifact)) { - // Should throw here - throw new RuntimeException("Invalid artifact to load"); - } - - // :[:[:]]: - int numSplits = artifact.split(":").length; - if (numSplits >= 3 && numSplits <= 6) { - return loadFromMvn(artifact, excludes, addSparkContext); - } else { - loadFromFs(artifact, addSparkContext); - LinkedList libs = new LinkedList(); - libs.add(artifact); - return libs; - } - } - - private void loadFromFs(String artifact, boolean addSparkContext) throws Exception { - File jarFile = new File(artifact); - - intp.interp().interpreter().global().new Run(); - - if (sc.version().startsWith("1.1")) { - updateRuntimeClassPath_1_x(new URL[] {jarFile.toURI().toURL()}); - } else { - updateRuntimeClassPath_2_x(new URL[] {jarFile.toURI().toURL()}); - } - - if (addSparkContext) { - sc.addJar(jarFile.getAbsolutePath()); - } - } - - private List loadFromMvn(String artifact, Collection excludes, - boolean addSparkContext) throws Exception { - List loadedLibs = new LinkedList(); - Collection allExclusions = new LinkedList(); - allExclusions.addAll(excludes); - allExclusions.addAll(Arrays.asList(exclusions)); - - List listOfArtifact; - listOfArtifact = getArtifactsWithDep(artifact, allExclusions); - - Iterator it = listOfArtifact.iterator(); - while (it.hasNext()) { - Artifact a = it.next().getArtifact(); - String gav = a.getGroupId() + ":" + a.getArtifactId() + ":" + a.getVersion(); - for (String exclude : allExclusions) { - if (gav.startsWith(exclude)) { - it.remove(); - break; - } - } - } - - List newClassPathList = new LinkedList(); - List files = new LinkedList(); - for (ArtifactResult artifactResult : listOfArtifact) { - logger.info("Load " + artifactResult.getArtifact().getGroupId() + ":" - + artifactResult.getArtifact().getArtifactId() + ":" - + artifactResult.getArtifact().getVersion()); - newClassPathList.add(artifactResult.getArtifact().getFile().toURI().toURL()); - files.add(artifactResult.getArtifact().getFile()); - loadedLibs.add(artifactResult.getArtifact().getGroupId() + ":" - + artifactResult.getArtifact().getArtifactId() + ":" - + artifactResult.getArtifact().getVersion()); - } - - intp.interp().interpreter().global().new Run(); - if (sc.version().startsWith("1.1")) { - updateRuntimeClassPath_1_x(newClassPathList.toArray(new URL[0])); - } else { - updateRuntimeClassPath_2_x(newClassPathList.toArray(new URL[0])); - } - updateCompilerClassPath(newClassPathList.toArray(new URL[0])); - - if (addSparkContext) { - for (File f : files) { - sc.addJar(f.getAbsolutePath()); - } - } - - return loadedLibs; - } - - /** - * @param dependency - * @param excludes list of pattern can either be of the form groupId:artifactId - * @return - * @throws Exception - */ - @Override - public List getArtifactsWithDep(String dependency, - Collection excludes) throws Exception { - Artifact artifact = new DefaultArtifact(inferScalaVersion(dependency)); - DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE); - PatternExclusionsDependencyFilter exclusionFilter = - new PatternExclusionsDependencyFilter(inferScalaVersion(excludes)); - - CollectRequest collectRequest = new CollectRequest(); - collectRequest.setRoot(new Dependency(artifact, JavaScopes.COMPILE)); - - synchronized (repos) { - for (RemoteRepository repo : repos) { - collectRequest.addRepository(repo); - } - } - DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, - DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter)); - return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); - } - - public static Collection inferScalaVersion(Collection artifact) { - List list = new LinkedList(); - for (String a : artifact) { - list.add(inferScalaVersion(a)); - } - return list; - } - - public static String inferScalaVersion(String artifact) { - int pos = artifact.indexOf(":"); - if (pos < 0 || pos + 2 >= artifact.length()) { - // failed to infer - return artifact; - } - - if (':' == artifact.charAt(pos + 1)) { - String restOfthem = ""; - String versionSep = ":"; - - String groupId = artifact.substring(0, pos); - int nextPos = artifact.indexOf(":", pos + 2); - if (nextPos < 0) { - if (artifact.charAt(artifact.length() - 1) == '*') { - nextPos = artifact.length() - 1; - versionSep = ""; - restOfthem = "*"; - } else { - versionSep = ""; - nextPos = artifact.length(); - } - } - - String artifactId = artifact.substring(pos + 2, nextPos); - if (nextPos < artifact.length()) { - if (!restOfthem.equals("*")) { - restOfthem = artifact.substring(nextPos + 1); - } - } - - String [] version = scala.util.Properties.versionNumberString().split("[.]"); - String scalaVersion = version[0] + "." + version[1]; - - return groupId + ":" + artifactId + "_" + scalaVersion + versionSep + restOfthem; - } else { - return artifact; - } - } -} diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index 90f311ed5c3..fd946c2d692 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -24,8 +24,6 @@ import java.util.LinkedList; import java.util.Properties; -import org.apache.spark.HttpServer; -import org.apache.spark.SecurityManager; import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.zeppelin.display.AngularObjectRegistry; diff --git a/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java b/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java index a0271f4471d..b226a001d24 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java @@ -19,7 +19,6 @@ import static org.junit.Assert.assertEquals; -import org.apache.zeppelin.spark.dep.SparkDependencyResolver; import org.junit.Test; public class SparkDependencyResolverTest { From b9e0e1ea285675d937d2fae8d2e912699a9f9eaa Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 9 Jun 2016 17:40:39 -0700 Subject: [PATCH 06/31] scala 2.11 support for spark interpreter --- .../apache/zeppelin/spark/DepInterpreter.java | 17 ++- .../zeppelin/spark/SparkInterpreter.java | 121 +++++++++++++----- .../zeppelin/spark/SparkInterpreterTest.java | 4 + 3 files changed, 107 insertions(+), 35 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java index 6e72d3af27a..83278bf59a6 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java @@ -52,9 +52,11 @@ import scala.None; import scala.Some; import scala.collection.convert.WrapAsJava$; +import scala.collection.JavaConversions; import scala.tools.nsc.Settings; import scala.tools.nsc.interpreter.Completion.Candidates; import scala.tools.nsc.interpreter.Completion.ScalaCompleter; +import scala.tools.nsc.interpreter.IMain; import scala.tools.nsc.interpreter.Results; import scala.tools.nsc.settings.MutableSettings.BooleanSetting; import scala.tools.nsc.settings.MutableSettings.PathSetting; @@ -180,7 +182,12 @@ private void createIMain() { new Object[]{intp}); } interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); - Map binder = (Map) getValue("_binder"); + Map binder; + if (isScala2_10()) { + binder = (Map) getValue("_binder"); + } else { + binder = (Map) getLastObject(); + } binder.put("depc", depc); interpret("@transient val z = " @@ -208,6 +215,13 @@ public Object getValue(String name) { } } + public Object getLastObject() { + IMain.Request r = (IMain.Request) invokeMethod(intp, "lastRequest"); + Object obj = r.lineRep().call("$result", + JavaConversions.asScalaBuffer(new LinkedList())); + return obj; + } + @Override public InterpreterResult interpret(String st, InterpreterContext context) { PrintStream printStream = new PrintStream(out); @@ -285,7 +299,6 @@ public List completion(String buf, int cursor) { } else { return new LinkedList(); } - } private List currentClassPath() { diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index bce72245660..5f5ff79670f 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -39,12 +39,14 @@ import org.apache.spark.SparkContext; import org.apache.spark.SparkEnv; +import org.apache.spark.SecurityManager; import org.apache.spark.repl.SparkILoop; import org.apache.spark.scheduler.ActiveJob; import org.apache.spark.scheduler.DAGScheduler; import org.apache.spark.scheduler.Pool; import org.apache.spark.sql.SQLContext; import org.apache.spark.ui.jobs.JobProgressListener; +import org.apache.spark.util.Utils; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -78,6 +80,7 @@ import scala.tools.nsc.Settings; import scala.tools.nsc.interpreter.Completion.Candidates; import scala.tools.nsc.interpreter.Completion.ScalaCompleter; +import scala.tools.nsc.interpreter.IMain; import scala.tools.nsc.interpreter.Results; import scala.tools.nsc.settings.MutableSettings; import scala.tools.nsc.settings.MutableSettings.BooleanSetting; @@ -115,6 +118,8 @@ public class SparkInterpreter extends Interpreter { private Map binder; private SparkVersion sparkVersion; + private File outputDir; // class outputdir for scala 2.11 + private HttpServer classServer; // classserver for scala 2.11 public SparkInterpreter(Properties property) { @@ -282,6 +287,19 @@ public SparkContext createSparkContext() { } } + + if (isScala2_11()) { + SparkConf conf = new SparkConf(); + classServer = new HttpServer( + conf, + outputDir, + new SecurityManager(conf), + 0, + "HTTP server"); + classServer.start(); + classServerUri = classServer.uri(); + } + SparkConf conf = new SparkConf() .setMaster(getProperty("master")) @@ -413,26 +431,49 @@ public void open() { * getClass.getClassLoader >> } >> in.setContextClassLoader() */ Settings settings = new Settings(); - if (getProperty("args") != null) { - String[] argsArray = getProperty("args").split(" "); - LinkedList argList = new LinkedList(); - for (String arg : argsArray) { - argList.add(arg); - } + // process args + String args = getProperty("args"); + if (args == null) { + args = ""; + } + + String[] argsArray = args.split(" "); + LinkedList argList = new LinkedList(); + for (String arg : argsArray) { + argList.add(arg); + } + + if (isScala2_10()) { scala.collection.immutable.List list = JavaConversions.asScalaBuffer(argList).toList(); - if (isScala2_10()) { - Object sparkCommandLine = instantiateClass( - "org.apache.spark.repl.SparkCommandLine", - new Class[]{ list.getClass() }, - new Object[]{ list }); + Object sparkCommandLine = instantiateClass( + "org.apache.spark.repl.SparkCommandLine", + new Class[]{ list.getClass() }, + new Object[]{ list }); - settings = (Settings) invokeMethod(sparkCommandLine, "settings"); - } else { - settings.processArguments(list, true); + settings = (Settings) invokeMethod(sparkCommandLine, "settings"); + } else { + String sparkReplClassDir = getProperty("spark.repl.classdir"); + if (sparkReplClassDir == null) { + sparkReplClassDir = System.getProperty("spark.repl.classdir"); + } + if (sparkReplClassDir == null) { + sparkReplClassDir = System.getProperty("java.io.tmpdir"); } + + outputDir = Utils.createTempDir(sparkReplClassDir, "classdir"); + + argList.add("-Yrepl-class-based"); + argList.add("-Yrepl-outdir"); + argList.add(outputDir.getAbsolutePath()); + + + scala.collection.immutable.List list = + JavaConversions.asScalaBuffer(argList).toList(); + + settings.processArguments(list, true); } // set classpath for scala compiler @@ -526,24 +567,22 @@ public void open() { if (isScala2_10()) { invokeMethod(intp, "setContextClassLoader"); invokeMethod(intp, "initializeSynchronous"); - } - if (classOutputDir == null) { - classOutputDir = settings.outputDirs().getSingleOutput().get(); - } else { - // change SparkIMain class output dir - settings.outputDirs().setSingleOutput(classOutputDir); - ClassLoader cl = (ClassLoader) invokeMethod(intp, "classLoader"); - try { - Field rootField = cl.getClass().getSuperclass().getDeclaredField("root"); - rootField.setAccessible(true); - rootField.set(cl, classOutputDir); - } catch (NoSuchFieldException | IllegalAccessException e) { - logger.error(e.getMessage(), e); + if (classOutputDir == null) { + classOutputDir = settings.outputDirs().getSingleOutput().get(); + } else { + // change SparkIMain class output dir + settings.outputDirs().setSingleOutput(classOutputDir); + ClassLoader cl = (ClassLoader) invokeMethod(intp, "classLoader"); + try { + Field rootField = cl.getClass().getSuperclass().getDeclaredField("root"); + rootField.setAccessible(true); + rootField.set(cl, classOutputDir); + } catch (NoSuchFieldException | IllegalAccessException e) { + logger.error(e.getMessage(), e); + } } - } - if (isScala2_10()) { completor = instantiateClass( "SparkJLineCompletion", new Class[]{findClass("org.apache.spark.repl.SparkIMain")}, @@ -568,8 +607,8 @@ public void open() { z = new ZeppelinContext(sc, sqlc, null, dep, Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); - interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); - binder = (Map) getValue("_binder"); + interpret("@transient val _binder = new java.util.HashMap[String, Object]()"); + binder = (Map) getLastObject(); binder.put("sc", sc); binder.put("sqlc", sqlc); binder.put("z", z); @@ -769,9 +808,14 @@ private String getCompletionTargetString(String text, int cursor) { return resultCompletionText; } + /* + * this method doesn't work in scala 2.11 + * Somehow intp.valueOfTerm returns scala.None always with -Yrepl-class-based option + */ public Object getValue(String name) { Object ret = invokeMethod(intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); - if (ret instanceof None) { + + if (ret instanceof None || ret instanceof scala.None$) { return null; } else if (ret instanceof Some) { return ((Some) ret).get(); @@ -780,6 +824,13 @@ public Object getValue(String name) { } } + public Object getLastObject() { + IMain.Request r = (IMain.Request) invokeMethod(intp, "lastRequest"); + Object obj = r.lineRep().call("$result", + JavaConversions.asScalaBuffer(new LinkedList())); + return obj; + } + String getJobGroup(InterpreterContext context){ return "zeppelin-" + context.getParagraphId(); } @@ -1049,6 +1100,10 @@ public void close() { if (numReferenceOfSparkContext.decrementAndGet() == 0) { sc.stop(); sc = null; + if (classServer != null) { + classServer.stop(); + classServer = null; + } } invokeMethod(intp, "close"); @@ -1144,7 +1199,7 @@ private boolean isScala2_10() { } private boolean isScala2_11() { - return !isScala2_11(); + return !isScala2_10(); } diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index fd946c2d692..29f958cc3fc 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.*; +import java.io.BufferedReader; import java.io.File; import java.util.HashMap; import java.util.LinkedList; @@ -26,6 +27,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; +import org.apache.spark.repl.SparkILoop; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.resource.LocalResourcePool; import org.apache.zeppelin.user.AuthenticationInfo; @@ -39,6 +41,7 @@ import org.junit.runners.MethodSorters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.tools.nsc.interpreter.IMain; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class SparkInterpreterTest { @@ -137,6 +140,7 @@ public void testBasicIntp() { assertEquals(InterpreterResult.Code.INCOMPLETE, incomplete.code()); assertTrue(incomplete.message().length() > 0); // expecting some error // message + /* * assertEquals(1, repl.getValue("a")); assertEquals(2, repl.getValue("b")); * repl.interpret("val ver = sc.version"); From 112ae7d13a362c8cf136812abb46827216817ccd Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 9 Jun 2016 19:29:36 -0700 Subject: [PATCH 07/31] Fix some reflections --- .../java/org/apache/zeppelin/spark/DepInterpreter.java | 2 +- .../java/org/apache/zeppelin/spark/SparkInterpreter.java | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java index 83278bf59a6..4f003dc5079 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java @@ -177,7 +177,7 @@ private void createIMain() { getProperty("zeppelin.dep.additionalRemoteRepository")); if (isScala2_10()) { completor = instantiateClass( - "SparkJLineCompletion", + "org.apache.spark.repl.SparkJLineCompletion", new Class[]{findClass("org.apache.spark.repl.SparkIMain")}, new Object[]{intp}); } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 5f5ff79670f..cb207a9213e 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -608,7 +608,12 @@ public void open() { Integer.parseInt(getProperty("zeppelin.spark.maxResult"))); interpret("@transient val _binder = new java.util.HashMap[String, Object]()"); - binder = (Map) getLastObject(); + Map binder; + if (isScala2_10()) { + binder = (Map) getValue("_binder"); + } else { + binder = (Map) getLastObject(); + } binder.put("sc", sc); binder.put("sqlc", sqlc); binder.put("z", z); From 222e4e7e811afd3aaa3e036a3608f59a641bda97 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 9 Jun 2016 19:57:57 -0700 Subject: [PATCH 08/31] Fix reflection on creating SparkCommandLine --- .../main/java/org/apache/zeppelin/spark/SparkInterpreter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index cb207a9213e..9f097bf258f 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -450,7 +450,7 @@ public void open() { Object sparkCommandLine = instantiateClass( "org.apache.spark.repl.SparkCommandLine", - new Class[]{ list.getClass() }, + new Class[]{ scala.collection.immutable.List.class }, new Object[]{ list }); settings = (Settings) invokeMethod(sparkCommandLine, "settings"); From dfe6e8368c5264770525879d3cec70dc9d502316 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 10 Jun 2016 12:40:07 -0700 Subject: [PATCH 09/31] Fix reflection around HttpServer and createTempDir --- .../zeppelin/spark/SparkInterpreter.java | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 9f097bf258f..e5e977f66e1 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -289,13 +289,7 @@ public SparkContext createSparkContext() { if (isScala2_11()) { - SparkConf conf = new SparkConf(); - classServer = new HttpServer( - conf, - outputDir, - new SecurityManager(conf), - 0, - "HTTP server"); + classServer = createHttpServer(outputDir); classServer.start(); classServerUri = classServer.uri(); } @@ -463,7 +457,7 @@ public void open() { sparkReplClassDir = System.getProperty("java.io.tmpdir"); } - outputDir = Utils.createTempDir(sparkReplClassDir, "classdir"); + outputDir = createTempDir(sparkReplClassDir); argList.add("-Yrepl-class-based"); argList.add("-Yrepl-outdir"); @@ -1216,4 +1210,40 @@ private Class findClass(String name) { return null; } } + + private File createTempDir(String dir) { + try { + return (File) invokeStaticMethod( + Utils.class, + "createTempDir", + new Class[]{String.class}, + new Object[]{dir}); + } catch (Exception e) { + return (File) invokeStaticMethod( + Utils.class, + "createTempDir", + new Class[]{String.class, String.class}, + new Object[]{dir, "spark"}); + } + } + + private HttpServer createHttpServer(File outputDir) { + SparkConf conf = new SparkConf(); + + try { + return (HttpServer) instantiateClass( + HttpServer.class.getName(), + new Class[] { File.class, SecurityManager.class, Integer.class, String.class}, + new Object[] { outputDir, new SecurityManager(conf), 0, "HTTP Server"} + ); + } catch (Exception e) { + return (HttpServer) instantiateClass( + HttpServer.class.getName(), + new Class[] { SparkConf.class, File.class, SecurityManager.class, Integer.class, + String.class}, + new Object[] { conf, outputDir, new SecurityManager(conf), 0, "HTTP Server"} + ); + } + } + } From c999a2defbef97ff48e579c83c767f2f2f3e1c60 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 10 Jun 2016 22:55:49 -0700 Subject: [PATCH 10/31] fix style --- .../main/java/org/apache/zeppelin/spark/SparkInterpreter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index e5e977f66e1..7fcffa5ac65 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -1240,7 +1240,7 @@ private HttpServer createHttpServer(File outputDir) { return (HttpServer) instantiateClass( HttpServer.class.getName(), new Class[] { SparkConf.class, File.class, SecurityManager.class, Integer.class, - String.class}, + String.class}, new Object[] { conf, outputDir, new SecurityManager(conf), 0, "HTTP Server"} ); } From 2ec51a31cc252e453ae516fb8564f690e6c5e245 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 13 Jun 2016 10:24:52 -0700 Subject: [PATCH 11/31] Fix reflection --- .../zeppelin/spark/SparkInterpreter.java | 87 +++++++++---------- 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 7fcffa5ac65..58bb2de6980 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -1138,11 +1138,7 @@ private Object invokeMethod(Object o, String name) { private Object invokeMethod(Object o, String name, Class [] argTypes, Object [] params) { try { return o.getClass().getMethod(name, argTypes).invoke(o, params); - } catch (NoSuchMethodException e) { - logger.error(e.getMessage(), e); - } catch (InvocationTargetException e) { - logger.error(e.getMessage(), e); - } catch (IllegalAccessException e) { + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { logger.error(e.getMessage(), e); } @@ -1156,12 +1152,7 @@ private Object invokeStaticMethod(Class c, String name) { private Object invokeStaticMethod(Class c, String name, Class [] argTypes, Object [] params) { try { return c.getMethod(name, argTypes).invoke(null, params); - } catch (NoSuchMethodException e) { - logger.error(e.getMessage(), e); - } catch (InvocationTargetException e) { - logger.error(e.getMessage(), e); - e.printStackTrace(); - } catch (IllegalAccessException e) { + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { logger.error(e.getMessage(), e); } @@ -1173,15 +1164,8 @@ private Object instantiateClass(String name, Class [] argTypes, Object [] params Constructor constructor = getClass().getClassLoader() .loadClass(name).getConstructor(argTypes); return constructor.newInstance(params); - } catch (NoSuchMethodException e) { - logger.error(e.getMessage(), e); - } catch (ClassNotFoundException e) { - logger.error(e.getMessage(), e); - } catch (IllegalAccessException e) { - logger.error(e.getMessage(), e); - } catch (InstantiationException e) { - logger.error(e.getMessage(), e); - } catch (InvocationTargetException e) { + } catch (NoSuchMethodException | ClassNotFoundException | IllegalAccessException | + InstantiationException | InvocationTargetException e) { logger.error(e.getMessage(), e); } return null; @@ -1212,38 +1196,53 @@ private Class findClass(String name) { } private File createTempDir(String dir) { + // try Utils.createTempDir() try { - return (File) invokeStaticMethod( - Utils.class, - "createTempDir", - new Class[]{String.class}, - new Object[]{dir}); - } catch (Exception e) { - return (File) invokeStaticMethod( - Utils.class, + return (File) Utils.class.getMethod( "createTempDir", - new Class[]{String.class, String.class}, - new Object[]{dir, "spark"}); + new Class[]{String.class, String.class}) + .invoke(null, new Object[]{dir, "spark"}); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + // fallback to old method + try { + return (File) Utils.class.getMethod( + "createTempDir", + new Class[]{String.class}) + .invoke(null, new Object[]{dir}); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e1) { + logger.error(e1.getMessage(), e1); + return null; + } } } private HttpServer createHttpServer(File outputDir) { SparkConf conf = new SparkConf(); - try { - return (HttpServer) instantiateClass( - HttpServer.class.getName(), - new Class[] { File.class, SecurityManager.class, Integer.class, String.class}, - new Object[] { outputDir, new SecurityManager(conf), 0, "HTTP Server"} - ); - } catch (Exception e) { - return (HttpServer) instantiateClass( - HttpServer.class.getName(), - new Class[] { SparkConf.class, File.class, SecurityManager.class, Integer.class, - String.class}, - new Object[] { conf, outputDir, new SecurityManager(conf), 0, "HTTP Server"} - ); + // try to create HttpServer + Constructor constructor = getClass().getClassLoader() + .loadClass(HttpServer.class.getName()) + .getConstructor(new Class[]{ + SparkConf.class, File.class, SecurityManager.class, int.class, String.class}); + + return (HttpServer) constructor.newInstance(new Object[] { + conf, outputDir, new SecurityManager(conf), 0, "HTTP Server"}); + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | + InstantiationException | InvocationTargetException e) { + // fallback to old constructor + Constructor constructor = null; + try { + constructor = getClass().getClassLoader() + .loadClass(HttpServer.class.getName()) + .getConstructor(new Class[]{ + File.class, SecurityManager.class, int.class, String.class}); + return (HttpServer) constructor.newInstance(new Object[] { + outputDir, new SecurityManager(conf), 0, "HTTP Server"}); + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | + InstantiationException | InvocationTargetException e1) { + logger.error(e1.getMessage(), e1); + return null; + } } } - } From 94247697a3a99877540bf59fa628c0fc8a3436c3 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 13 Jun 2016 10:31:39 -0700 Subject: [PATCH 12/31] style --- .../java/org/apache/zeppelin/spark/SparkInterpreter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 58bb2de6980..140013d1b06 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -1223,10 +1223,10 @@ private HttpServer createHttpServer(File outputDir) { Constructor constructor = getClass().getClassLoader() .loadClass(HttpServer.class.getName()) .getConstructor(new Class[]{ - SparkConf.class, File.class, SecurityManager.class, int.class, String.class}); + SparkConf.class, File.class, SecurityManager.class, int.class, String.class}); return (HttpServer) constructor.newInstance(new Object[] { - conf, outputDir, new SecurityManager(conf), 0, "HTTP Server"}); + conf, outputDir, new SecurityManager(conf), 0, "HTTP Server"}); } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) { // fallback to old constructor @@ -1235,9 +1235,9 @@ private HttpServer createHttpServer(File outputDir) { constructor = getClass().getClassLoader() .loadClass(HttpServer.class.getName()) .getConstructor(new Class[]{ - File.class, SecurityManager.class, int.class, String.class}); + File.class, SecurityManager.class, int.class, String.class}); return (HttpServer) constructor.newInstance(new Object[] { - outputDir, new SecurityManager(conf), 0, "HTTP Server"}); + outputDir, new SecurityManager(conf), 0, "HTTP Server"}); } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e1) { logger.error(e1.getMessage(), e1); From 6b9ff1d654b0876be74b9ad0086260e1f2958145 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 13 Jun 2016 11:10:19 -0700 Subject: [PATCH 13/31] SparkContext sharing seems not working in scala 2.11, disable the test --- .../zeppelin/spark/SparkInterpreter.java | 4 +- .../zeppelin/spark/SparkInterpreterTest.java | 58 ++++++++++--------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 140013d1b06..897afd879a8 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -1172,7 +1172,7 @@ private Object instantiateClass(String name, Class [] argTypes, Object [] params } // function works after intp is initialized - private boolean isScala2_10() { + boolean isScala2_10() { try { this.getClass().forName("org.apache.spark.repl.SparkIMain"); return true; @@ -1181,7 +1181,7 @@ private boolean isScala2_10() { } } - private boolean isScala2_11() { + boolean isScala2_11() { return !isScala2_10(); } diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index 29f958cc3fc..824be1432e2 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -178,21 +178,23 @@ public void testListener() { @Test public void testSparkSql(){ - repl.interpret("case class Person(name:String, age:Int)\n", context); - repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context); - assertEquals(Code.SUCCESS, repl.interpret("people.take(3)", context).code()); - - - if (getSparkVersionNumber() <= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm by default. - // create new interpreter - Properties p = new Properties(); - SparkInterpreter repl2 = new SparkInterpreter(p); - repl2.open(); - - repl.interpret("case class Man(name:String, age:Int)", context); - repl.interpret("val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))", context); - assertEquals(Code.SUCCESS, repl.interpret("man.take(3)", context).code()); - repl2.getSparkContext().stop(); + if (repl.isScala2_10()) { + repl.interpret("case class Person(name:String, age:Int)\n", context); + repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context); + assertEquals(Code.SUCCESS, repl.interpret("people.take(3)", context).code()); + + + if (getSparkVersionNumber() <= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm by default. + // create new interpreter + Properties p = new Properties(); + SparkInterpreter repl2 = new SparkInterpreter(p); + repl2.open(); + + repl.interpret("case class Man(name:String, age:Int)", context); + repl.interpret("val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))", context); + assertEquals(Code.SUCCESS, repl.interpret("man.take(3)", context).code()); + repl2.getSparkContext().stop(); + } } } @@ -219,18 +221,20 @@ public void emptyConfigurationVariablesOnlyForNonSparkProperties() { @Test public void shareSingleSparkContext() throws InterruptedException { - // create another SparkInterpreter - SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties()); - repl2.setInterpreterGroup(intpGroup); - intpGroup.get("note").add(repl2); - repl2.open(); - - assertEquals(Code.SUCCESS, - repl.interpret("print(sc.parallelize(1 to 10).count())", context).code()); - assertEquals(Code.SUCCESS, - repl2.interpret("print(sc.parallelize(1 to 10).count())", context).code()); - - repl2.close(); + if (repl.isScala2_10()) { + // create another SparkInterpreter + SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties()); + repl2.setInterpreterGroup(intpGroup); + intpGroup.get("note").add(repl2); + repl2.open(); + + assertEquals(Code.SUCCESS, + repl.interpret("print(sc.parallelize(1 to 10).count())", context).code()); + assertEquals(Code.SUCCESS, + repl2.interpret("print(sc.parallelize(1 to 10).count())", context).code()); + + repl2.close(); + } } @Test From 6d3e7e2cead44d51387700d2802cc8866d3480a4 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 13 Jun 2016 12:10:20 -0700 Subject: [PATCH 14/31] Update FlinkInterpreter --- .../zeppelin/flink/FlinkInterpreter.java | 72 ++++++++----------- 1 file changed, 28 insertions(+), 44 deletions(-) diff --git a/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java b/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java index 18457700706..305c3a50c88 100644 --- a/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java +++ b/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java @@ -43,6 +43,7 @@ import scala.Console; import scala.None; import scala.Some; +import scala.collection.JavaConversions; import scala.collection.immutable.Nil; import scala.runtime.AbstractFunction0; import scala.tools.nsc.Settings; @@ -93,7 +94,7 @@ public void open() { // prepare bindings imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); - binder = (Map) getValue("_binder"); + Map binder = (Map) getLastObject(); // import libraries imain.interpret("import scala.tools.nsc.io._"); @@ -103,39 +104,9 @@ public void open() { imain.interpret("import org.apache.flink.api.scala._"); imain.interpret("import org.apache.flink.api.common.functions._"); - String scalaVersion = scalaVersion(imain); - // scala 2.10 use imain.bindValue("env" env) - // scala 2.11 use imain.put("env", env); - String bindMethod = "bindValue"; - if (scalaVersion.equals("2.11")) { - bindMethod = "put"; - } - - java.lang.reflect.Method method; - try { - method = imain.getClass().getMethod(bindMethod, String.class, Object.class); - if (method != null) { - method.invoke(imain, "env", env); - } - } catch (Exception e) { - if (logger.isDebugEnabled()) { - logger.debug("Error binding environment variable: " + e.getMessage(), e); - } - } - } - - private String scalaVersion(IMain imain) { - String version = "2.10"; - - try { - if (imain.getClass().getMethod("put", String.class, Object.class) != null) { - version = "2.11"; - } - } catch (Exception e) { - // ignore - } - - return version; + binder.put("env", env); + imain.interpret("val env = _binder.get(\"env\").asInstanceOf[" + + env.getClass().getName() + "]"); } private boolean localMode() { @@ -224,16 +195,11 @@ private List classPath(ClassLoader cl) { return paths; } - public Object getValue(String name) { - IMain imain = flinkIloop.intp(); - Object ret = imain.valueOfTerm(name); - if (ret instanceof None) { - return null; - } else if (ret instanceof Some) { - return ((Some) ret).get(); - } else { - return ret; - } + public Object getLastObject() { + Object obj = imain.lastRequest().lineRep().call( + "$result", + JavaConversions.asScalaBuffer(new LinkedList())); + return obj; } @Override @@ -388,4 +354,22 @@ private void stopFlinkMiniCluster() { static final String toString(Object o) { return (o instanceof String) ? (String) o : ""; } + + private Object invokeMethod(Object o, String name) { + return invokeMethod(o, name, new Class[]{}, new Object[]{}); + } + + private Object invokeMethod(Object o, String name, Class [] argTypes, Object [] params) { + try { + return o.getClass().getMethod(name, argTypes).invoke(o, params); + } catch (NoSuchMethodException e) { + logger.error(e.getMessage(), e); + } catch (InvocationTargetException e) { + logger.error(e.getMessage(), e); + } catch (IllegalAccessException e) { + logger.error(e.getMessage(), e); + } + + return null; + } } From fc9e8a06b252655fa3bd9006364a02f476c48cea Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 13 Jun 2016 12:34:19 -0700 Subject: [PATCH 15/31] Update ignite interpreter --- .../zeppelin/ignite/IgniteInterpreter.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java b/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java index fa5a079ae19..3f240183567 100644 --- a/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java +++ b/ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java @@ -44,6 +44,7 @@ import scala.Console; import scala.None; import scala.Some; +import scala.collection.JavaConversions; import scala.tools.nsc.Settings; import scala.tools.nsc.interpreter.IMain; import scala.tools.nsc.interpreter.Results.Result; @@ -174,16 +175,11 @@ private List classPath(ClassLoader cl) { return paths; } - public Object getValue(String name) { - Object val = imain.valueOfTerm(name); - - if (val instanceof None) { - return null; - } else if (val instanceof Some) { - return ((Some) val).get(); - } else { - return val; - } + public Object getLastObject() { + Object obj = imain.lastRequest().lineRep().call( + "$result", + JavaConversions.asScalaBuffer(new LinkedList())); + return obj; } private Ignite getIgnite() { @@ -222,7 +218,7 @@ private Ignite getIgnite() { private void initIgnite() { imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); - Map binder = (Map) getValue("_binder"); + Map binder = (Map) getLastObject(); if (getIgnite() != null) { binder.put("ignite", ignite); From 9f5d2a2fa7394ffa1a5cd0a6ca5e1912b1458794 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 13 Jun 2016 12:51:27 -0700 Subject: [PATCH 16/31] Remove unused methods --- .../zeppelin/flink/FlinkInterpreter.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java b/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java index 305c3a50c88..d3229cf09a8 100644 --- a/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java +++ b/flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java @@ -354,22 +354,4 @@ private void stopFlinkMiniCluster() { static final String toString(Object o) { return (o instanceof String) ? (String) o : ""; } - - private Object invokeMethod(Object o, String name) { - return invokeMethod(o, name, new Class[]{}, new Object[]{}); - } - - private Object invokeMethod(Object o, String name, Class [] argTypes, Object [] params) { - try { - return o.getClass().getMethod(name, argTypes).invoke(o, params); - } catch (NoSuchMethodException e) { - logger.error(e.getMessage(), e); - } catch (InvocationTargetException e) { - logger.error(e.getMessage(), e); - } catch (IllegalAccessException e) { - logger.error(e.getMessage(), e); - } - - return null; - } } From 74d8a623d712d97494250c6b8c22fcf6caf0be56 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Wed, 15 Jun 2016 23:47:09 -0700 Subject: [PATCH 17/31] Force close Closes lresende/incubator-zeppelin/#1 --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index fce68ee1f78..498c00f258c 100755 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,6 @@ - org.slf4j slf4j-api From 736d055c07e3f05c6c929dcf0dc0e9104ea735cc Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 29 Jun 2016 16:09:15 -0700 Subject: [PATCH 18/31] make binary built with scala 2.11 work with spark_2.10 binary Closes lresende/incubator-zeppelin/#2 --- spark/pom.xml | 4 +-- .../apache/zeppelin/spark/DepInterpreter.java | 2 +- .../zeppelin/spark/SparkInterpreter.java | 10 +++---- .../zeppelin/rest/AbstractTestRestApi.java | 26 +++++++++++++++++++ 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/spark/pom.xml b/spark/pom.xml index ae2887ef6c1..712691e923e 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -52,11 +52,11 @@ slf4j-log4j12 - + ${project.groupId} diff --git a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java index 4f003dc5079..72550d15c0d 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java @@ -166,7 +166,7 @@ private void createIMain() { interpreter.createInterpreter(); - intp = interpreter.intp(); + intp = invokeMethod(interpreter, "intp"); if (isScala2_10()) { invokeMethod(intp, "setContextClassLoader"); diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 897afd879a8..58a7ee9084b 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -266,8 +266,8 @@ public SparkContext createSparkContext() { String classServerUri = null; try { // in case of spark 1.1x, spark 1.2x - Method classServer = interpreter.intp().getClass().getMethod("classServer"); - HttpServer httpServer = (HttpServer) classServer.invoke(interpreter.intp()); + Method classServer = intp.getClass().getMethod("classServer"); + HttpServer httpServer = (HttpServer) classServer.invoke(intp); classServerUri = httpServer.uri(); } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { @@ -276,8 +276,8 @@ public SparkContext createSparkContext() { if (classServerUri == null) { try { // for spark 1.3x - Method classServer = interpreter.intp().getClass().getMethod("classServerUri"); - classServerUri = (String) classServer.invoke(interpreter.intp()); + Method classServer = intp.getClass().getMethod("classServerUri"); + classServerUri = (String) classServer.invoke(intp); } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { // continue instead of: throw new InterpreterException(e); @@ -556,7 +556,7 @@ public void open() { interpreter.createInterpreter(); - intp = interpreter.intp(); + intp = invokeMethod(interpreter, "intp"); if (isScala2_10()) { invokeMethod(intp, "setContextClassLoader"); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index f2d4c99ca49..cb77ae745be 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -102,6 +102,30 @@ protected static void startUp() throws Exception { System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), "../"); System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_WAR.getVarName(), "../zeppelin-web/dist"); LOG.info("Staring test Zeppelin up..."); + + + // exclude org.apache.zeppelin.rinterpreter.* for scala 2.11 test + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + String interpreters = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS); + String interpretersCompatibleWithScala211Test = null; + + for (String intp : interpreters.split(",")) { + if (intp.startsWith("org.apache.zeppelin.rinterpreter")) { + continue; + } + + if (interpretersCompatibleWithScala211Test == null) { + interpretersCompatibleWithScala211Test = intp; + } else { + interpretersCompatibleWithScala211Test += "," + intp; + } + } + + System.setProperty( + ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), + interpretersCompatibleWithScala211Test); + + executor = Executors.newSingleThreadExecutor(); executor.submit(server); long s = System.currentTimeMillis(); @@ -241,6 +265,8 @@ protected static void shutDown() throws Exception { } LOG.info("Test Zeppelin terminated."); + + System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName()); } } From e0685930d50ce1dd3a3e35b7e414b1d863ed4374 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Thu, 7 Jul 2016 09:55:14 -0700 Subject: [PATCH 19/31] Fix pom.xml merge conflict --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 498c00f258c..560df065b2c 100755 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,6 @@ - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.maven-v4_0_0.xsd"> 4.0.0 From a3d052595824bcf7800b475595e758b6e677ecff Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Thu, 7 Jul 2016 11:03:23 -0700 Subject: [PATCH 20/31] Fix new code to support both scala versions --- .../java/org/apache/zeppelin/spark/SparkInterpreter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 58a7ee9084b..cbc0cd08eee 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -932,8 +932,8 @@ public InterpreterResult interpretInput(String[] lines, InterpreterContext conte // make sure code does not finish with comment if (r == Code.INCOMPLETE) { - scala.tools.nsc.interpreter.Results.Result res; - res = intp.interpret(incomplete + "\nprint(\"\")"); + scala.tools.nsc.interpreter.Results.Result res = null; + res = interpret(incomplete + "\nprint(\"\")"); r = getResultCode(res); } @@ -950,7 +950,7 @@ public InterpreterResult interpretInput(String[] lines, InterpreterContext conte } private void putLatestVarInResourcePool(InterpreterContext context) { - String varName = intp.mostRecentVar(); + String varName = (String) invokeMethod(intp, "mostRecentVar"); if (varName == null || varName.isEmpty()) { return; } From 6e4f7b017fdfda026cfee8f7441db7fe36b85616 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Fri, 8 Jul 2016 18:04:14 -0700 Subject: [PATCH 21/31] Force scala-library dependency version based on scala --- zeppelin-display/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/zeppelin-display/pom.xml b/zeppelin-display/pom.xml index 1f316fcd5f7..5cec0d121f2 100644 --- a/zeppelin-display/pom.xml +++ b/zeppelin-display/pom.xml @@ -81,6 +81,7 @@ org.scala-lang scala-library + ${scala.version} From 98790a6d8965d03f931577c2e99ee1c01a9285b4 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Fri, 8 Jul 2016 18:04:42 -0700 Subject: [PATCH 22/31] Remove obsolete/commented config --- pom.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pom.xml b/pom.xml index 560df065b2c..221adfe3607 100755 --- a/pom.xml +++ b/pom.xml @@ -643,8 +643,6 @@ 2.10.5 2.10 - From cbf84c7aa90b096373a219cd611da11c9171a275 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Fri, 8 Jul 2016 18:27:45 -0700 Subject: [PATCH 23/31] Force Scala 2.11 profile to be called --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 091f32a6fe1..4554c2c108c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: # Test all modules with scala 2.11 - jdk: "oraclejdk7" - env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Dscala-2.11 -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="-Dpython.test.exclude=''" + env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Dscala-2.11 -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="-Dpython.test.exclude=''" # Test spark module for 1.5.2 - jdk: "oraclejdk7" From 91942187dcada56bb924e0c367daab2862882545 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 9 Jul 2016 11:07:28 -0700 Subject: [PATCH 24/31] initialize imain --- .../zeppelin/spark/SparkInterpreter.java | 5 +- .../zeppelin/spark/SparkInterpreterTest.java | 54 +++++++++---------- 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index cbc0cd08eee..5904efcad02 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -557,11 +557,10 @@ public void open() { interpreter.createInterpreter(); intp = invokeMethod(interpreter, "intp"); + invokeMethod(intp, "setContextClassLoader"); + invokeMethod(intp, "initializeSynchronous"); if (isScala2_10()) { - invokeMethod(intp, "setContextClassLoader"); - invokeMethod(intp, "initializeSynchronous"); - if (classOutputDir == null) { classOutputDir = settings.outputDirs().getSingleOutput().get(); } else { diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index 824be1432e2..815e77f9bd9 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -178,23 +178,21 @@ public void testListener() { @Test public void testSparkSql(){ - if (repl.isScala2_10()) { - repl.interpret("case class Person(name:String, age:Int)\n", context); - repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context); - assertEquals(Code.SUCCESS, repl.interpret("people.take(3)", context).code()); - - - if (getSparkVersionNumber() <= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm by default. - // create new interpreter - Properties p = new Properties(); - SparkInterpreter repl2 = new SparkInterpreter(p); - repl2.open(); - - repl.interpret("case class Man(name:String, age:Int)", context); - repl.interpret("val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))", context); - assertEquals(Code.SUCCESS, repl.interpret("man.take(3)", context).code()); - repl2.getSparkContext().stop(); - } + repl.interpret("case class Person(name:String, age:Int)\n", context); + repl.interpret("val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n", context); + assertEquals(Code.SUCCESS, repl.interpret("people.take(3)", context).code()); + + + if (getSparkVersionNumber() <= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm by default. + // create new interpreter + Properties p = new Properties(); + SparkInterpreter repl2 = new SparkInterpreter(p); + repl2.open(); + + repl.interpret("case class Man(name:String, age:Int)", context); + repl.interpret("val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))", context); + assertEquals(Code.SUCCESS, repl.interpret("man.take(3)", context).code()); + repl2.getSparkContext().stop(); } } @@ -221,20 +219,18 @@ public void emptyConfigurationVariablesOnlyForNonSparkProperties() { @Test public void shareSingleSparkContext() throws InterruptedException { - if (repl.isScala2_10()) { - // create another SparkInterpreter - SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties()); - repl2.setInterpreterGroup(intpGroup); - intpGroup.get("note").add(repl2); - repl2.open(); + // create another SparkInterpreter + SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties()); + repl2.setInterpreterGroup(intpGroup); + intpGroup.get("note").add(repl2); + repl2.open(); - assertEquals(Code.SUCCESS, - repl.interpret("print(sc.parallelize(1 to 10).count())", context).code()); - assertEquals(Code.SUCCESS, - repl2.interpret("print(sc.parallelize(1 to 10).count())", context).code()); + assertEquals(Code.SUCCESS, + repl.interpret("print(sc.parallelize(1 to 10).count())", context).code()); + assertEquals(Code.SUCCESS, + repl2.interpret("print(sc.parallelize(1 to 10).count())", context).code()); - repl2.close(); - } + repl2.close(); } @Test From de4fc10be4d939a06fe182da44d8bdc8d0019790 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Mon, 11 Jul 2016 12:27:27 -0700 Subject: [PATCH 25/31] Minor change to force build --- pom.xml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 221adfe3607..cc679d47f35 100755 --- a/pom.xml +++ b/pom.xml @@ -388,12 +388,20 @@ - + From dd794434076362e5db7b4ef5a4f0a891eb6db7c7 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Mon, 11 Jul 2016 15:56:28 -0700 Subject: [PATCH 26/31] Minor formatting change to force build --- pom.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index cc679d47f35..3b99b344ed6 100755 --- a/pom.xml +++ b/pom.xml @@ -398,8 +398,13 @@ default verify - check - true + check + + + + + true + --> From 4e2237a607770b3033442029cc972a074435d9b0 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Tue, 12 Jul 2016 21:30:24 -0700 Subject: [PATCH 27/31] Update readme to use profile to build scala 2.11 and match CI --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5feab28399b..a1bfaca24b2 100644 --- a/README.md +++ b/README.md @@ -296,10 +296,10 @@ For configuration details check __`./conf`__ subdirectory. ### Building for Scala 2.11 -To produce a Zeppelin package compiled with Scala 2.11, use the -Dscala-2.11 property: +To produce a Zeppelin package compiled with Scala 2.11, use the -Pscala-2.11 profile: ``` -mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Dscala-2.11 -DskipTests clean install +mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Pscala-2.11 -DskipTests clean install ``` ### Package From 6e5e5ad83c7461af17e5b8a65075e257263ba864 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Tue, 12 Jul 2016 22:45:42 -0700 Subject: [PATCH 28/31] Refactor utility methods to helper class --- .../apache/zeppelin/spark/DepInterpreter.java | 109 +++--------------- .../zeppelin/spark/SparkInterpreter.java | 104 +++++------------ .../java/org/apache/zeppelin/spark/Utils.java | 92 +++++++++++++++ 3 files changed, 133 insertions(+), 172 deletions(-) create mode 100644 spark/src/main/java/org/apache/zeppelin/spark/Utils.java diff --git a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java index 72550d15c0d..5dc5d03d6d3 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java @@ -114,7 +114,7 @@ public static String getSystemDefault( @Override public void close() { if (intp != null) { - invokeMethod(intp, "close"); + Utils.invokeMethod(intp, "close"); } } @@ -166,24 +166,24 @@ private void createIMain() { interpreter.createInterpreter(); - intp = invokeMethod(interpreter, "intp"); + intp = Utils.invokeMethod(interpreter, "intp"); - if (isScala2_10()) { - invokeMethod(intp, "setContextClassLoader"); - invokeMethod(intp, "initializeSynchronous"); + if (Utils.isScala2_10()) { + Utils.invokeMethod(intp, "setContextClassLoader"); + Utils.invokeMethod(intp, "initializeSynchronous"); } depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"), getProperty("zeppelin.dep.additionalRemoteRepository")); - if (isScala2_10()) { - completor = instantiateClass( + if (Utils.isScala2_10()) { + completor = Utils.instantiateClass( "org.apache.spark.repl.SparkJLineCompletion", - new Class[]{findClass("org.apache.spark.repl.SparkIMain")}, + new Class[]{Utils.findClass("org.apache.spark.repl.SparkIMain")}, new Object[]{intp}); } interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); Map binder; - if (isScala2_10()) { + if (Utils.isScala2_10()) { binder = (Map) getValue("_binder"); } else { binder = (Map) getLastObject(); @@ -197,7 +197,7 @@ private void createIMain() { } private Results.Result interpret(String line) { - return (Results.Result) invokeMethod( + return (Results.Result) Utils.invokeMethod( intp, "interpret", new Class[] {String.class}, @@ -205,7 +205,8 @@ private Results.Result interpret(String line) { } public Object getValue(String name) { - Object ret = invokeMethod(intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); + Object ret = Utils.invokeMethod( + intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); if (ret instanceof None) { return null; } else if (ret instanceof Some) { @@ -216,7 +217,7 @@ public Object getValue(String name) { } public Object getLastObject() { - IMain.Request r = (IMain.Request) invokeMethod(intp, "lastRequest"); + IMain.Request r = (IMain.Request) Utils.invokeMethod(intp, "lastRequest"); Object obj = r.lineRep().call("$result", JavaConversions.asScalaBuffer(new LinkedList())); return obj; @@ -284,8 +285,8 @@ public int getProgress(InterpreterContext context) { @Override public List completion(String buf, int cursor) { - if (isScala2_10()) { - ScalaCompleter c = (ScalaCompleter) invokeMethod(completor, "completer"); + if (Utils.isScala2_10()) { + ScalaCompleter c = (ScalaCompleter) Utils.invokeMethod(completor, "completer"); Candidates ret = c.complete(buf, cursor); List candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates()); @@ -356,84 +357,4 @@ public Scheduler getScheduler() { return null; } } - - private Object invokeMethod(Object o, String name) { - return invokeMethod(o, name, new Class[]{}, new Object[]{}); - } - - private Object invokeMethod(Object o, String name, Class [] argTypes, Object [] params) { - try { - return o.getClass().getMethod(name, argTypes).invoke(o, params); - } catch (NoSuchMethodException e) { - logger.error(e.getMessage(), e); - } catch (InvocationTargetException e) { - logger.error(e.getMessage(), e); - } catch (IllegalAccessException e) { - logger.error(e.getMessage(), e); - } - - return null; - } - - private Object invokeStaticMethod(Class c, String name) { - return invokeStaticMethod(c, name, new Class[]{}, new Object[]{}); - } - - private Object invokeStaticMethod(Class c, String name, Class [] argTypes, Object [] params) { - try { - return c.getMethod(name, argTypes).invoke(null, params); - } catch (NoSuchMethodException e) { - logger.error(e.getMessage(), e); - } catch (InvocationTargetException e) { - logger.error(e.getMessage(), e); - e.printStackTrace(); - } catch (IllegalAccessException e) { - logger.error(e.getMessage(), e); - } - - return null; - } - - private Object instantiateClass(String name, Class [] argTypes, Object [] params) { - try { - Constructor constructor = getClass().getClassLoader() - .loadClass(name).getConstructor(argTypes); - return constructor.newInstance(params); - } catch (NoSuchMethodException e) { - logger.error(e.getMessage(), e); - } catch (ClassNotFoundException e) { - logger.error(e.getMessage(), e); - } catch (IllegalAccessException e) { - logger.error(e.getMessage(), e); - } catch (InstantiationException e) { - logger.error(e.getMessage(), e); - } catch (InvocationTargetException e) { - logger.error(e.getMessage(), e); - } - return null; - } - - // function works after intp is initialized - private boolean isScala2_10() { - try { - this.getClass().forName("org.apache.spark.repl.SparkIMain"); - return true; - } catch (ClassNotFoundException e) { - return false; - } - } - - private boolean isScala2_11() { - return !isScala2_11(); - } - - - private Class findClass(String name) { - try { - return this.getClass().forName(name); - } catch (ClassNotFoundException e) { - logger.error(e.getMessage(), e); - return null; - } - } } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 5904efcad02..44626bfeae6 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -46,7 +46,6 @@ import org.apache.spark.scheduler.Pool; import org.apache.spark.sql.SQLContext; import org.apache.spark.ui.jobs.JobProgressListener; -import org.apache.spark.util.Utils; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -230,8 +229,8 @@ public SQLContext getSQLContext() { public SparkDependencyResolver getDependencyResolver() { if (dep == null) { dep = new SparkDependencyResolver( - (Global) invokeMethod(intp, "global"), - (ClassLoader) invokeMethod(invokeMethod(intp, "classLoader"), "getParent"), + (Global) Utils.invokeMethod(intp, "global"), + (ClassLoader) Utils.invokeMethod(Utils.invokeMethod(intp, "classLoader"), "getParent"), sc, getProperty("zeppelin.dep.localrepo"), getProperty("zeppelin.dep.additionalRemoteRepository")); @@ -257,10 +256,11 @@ public SparkContext createSparkContext() { String execUri = System.getenv("SPARK_EXECUTOR_URI"); String[] jars = null; - if (isScala2_10()) { - jars = (String[]) invokeStaticMethod(SparkILoop.class, "getAddedJars"); + if (Utils.isScala2_10()) { + jars = (String[]) Utils.invokeStaticMethod(SparkILoop.class, "getAddedJars"); } else { - jars = (String[]) invokeStaticMethod(findClass("org.apache.spark.repl.Main"), "getAddedJars"); + jars = (String[]) Utils.invokeStaticMethod( + findClass("org.apache.spark.repl.Main"), "getAddedJars"); } String classServerUri = null; @@ -288,7 +288,7 @@ public SparkContext createSparkContext() { } - if (isScala2_11()) { + if (Utils.isScala2_11()) { classServer = createHttpServer(outputDir); classServer.start(); classServerUri = classServer.uri(); @@ -438,16 +438,16 @@ public void open() { argList.add(arg); } - if (isScala2_10()) { + if (Utils.isScala2_10()) { scala.collection.immutable.List list = JavaConversions.asScalaBuffer(argList).toList(); - Object sparkCommandLine = instantiateClass( + Object sparkCommandLine = Utils.instantiateClass( "org.apache.spark.repl.SparkCommandLine", new Class[]{ scala.collection.immutable.List.class }, new Object[]{ list }); - settings = (Settings) invokeMethod(sparkCommandLine, "settings"); + settings = (Settings) Utils.invokeMethod(sparkCommandLine, "settings"); } else { String sparkReplClassDir = getProperty("spark.repl.classdir"); if (sparkReplClassDir == null) { @@ -556,17 +556,17 @@ public void open() { interpreter.createInterpreter(); - intp = invokeMethod(interpreter, "intp"); - invokeMethod(intp, "setContextClassLoader"); - invokeMethod(intp, "initializeSynchronous"); + intp = Utils.invokeMethod(interpreter, "intp"); + Utils.invokeMethod(intp, "setContextClassLoader"); + Utils.invokeMethod(intp, "initializeSynchronous"); - if (isScala2_10()) { + if (Utils.isScala2_10()) { if (classOutputDir == null) { classOutputDir = settings.outputDirs().getSingleOutput().get(); } else { // change SparkIMain class output dir settings.outputDirs().setSingleOutput(classOutputDir); - ClassLoader cl = (ClassLoader) invokeMethod(intp, "classLoader"); + ClassLoader cl = (ClassLoader) Utils.invokeMethod(intp, "classLoader"); try { Field rootField = cl.getClass().getSuperclass().getDeclaredField("root"); rootField.setAccessible(true); @@ -576,7 +576,7 @@ public void open() { } } - completor = instantiateClass( + completor = Utils.instantiateClass( "SparkJLineCompletion", new Class[]{findClass("org.apache.spark.repl.SparkIMain")}, new Object[]{intp}); @@ -602,7 +602,7 @@ public void open() { interpret("@transient val _binder = new java.util.HashMap[String, Object]()"); Map binder; - if (isScala2_10()) { + if (Utils.isScala2_10()) { binder = (Map) getValue("_binder"); } else { binder = (Map) getLastObject(); @@ -643,7 +643,7 @@ public void open() { Integer.parseInt(getProperty("zeppelin.spark.maxResult")) + ")"); */ - if (isScala2_10()) { + if (Utils.isScala2_10()) { try { if (sparkVersion.oldLoadFilesMethodName()) { Method loadFiles = this.interpreter.getClass().getMethod("loadFiles", Settings.class); @@ -701,7 +701,7 @@ public void open() { } private Results.Result interpret(String line) { - return (Results.Result) invokeMethod( + return (Results.Result) Utils.invokeMethod( intp, "interpret", new Class[] {String.class}, @@ -747,8 +747,8 @@ public List completion(String buf, int cursor) { completionText = ""; cursor = completionText.length(); } - if (isScala2_10()) { - ScalaCompleter c = (ScalaCompleter) invokeMethod(completor, "completor"); + if (Utils.isScala2_10()) { + ScalaCompleter c = (ScalaCompleter) Utils.invokeMethod(completor, "completor"); Candidates ret = c.complete(completionText, cursor); List candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates()); @@ -811,7 +811,8 @@ private String getCompletionTargetString(String text, int cursor) { * Somehow intp.valueOfTerm returns scala.None always with -Yrepl-class-based option */ public Object getValue(String name) { - Object ret = invokeMethod(intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); + Object ret = Utils.invokeMethod( + intp, "valueOfTerm", new Class[]{String.class}, new Object[]{name}); if (ret instanceof None || ret instanceof scala.None$) { return null; @@ -823,7 +824,7 @@ public Object getValue(String name) { } public Object getLastObject() { - IMain.Request r = (IMain.Request) invokeMethod(intp, "lastRequest"); + IMain.Request r = (IMain.Request) Utils.invokeMethod(intp, "lastRequest"); Object obj = r.lineRep().call("$result", JavaConversions.asScalaBuffer(new LinkedList())); return obj; @@ -949,7 +950,7 @@ public InterpreterResult interpretInput(String[] lines, InterpreterContext conte } private void putLatestVarInResourcePool(InterpreterContext context) { - String varName = (String) invokeMethod(intp, "mostRecentVar"); + String varName = (String) Utils.invokeMethod(intp, "mostRecentVar"); if (varName == null || varName.isEmpty()) { return; } @@ -1104,7 +1105,7 @@ public void close() { } } - invokeMethod(intp, "close"); + Utils.invokeMethod(intp, "close"); } @Override @@ -1130,59 +1131,6 @@ public SparkVersion getSparkVersion() { return sparkVersion; } - private Object invokeMethod(Object o, String name) { - return invokeMethod(o, name, new Class[]{}, new Object[]{}); - } - - private Object invokeMethod(Object o, String name, Class [] argTypes, Object [] params) { - try { - return o.getClass().getMethod(name, argTypes).invoke(o, params); - } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { - logger.error(e.getMessage(), e); - } - - return null; - } - - private Object invokeStaticMethod(Class c, String name) { - return invokeStaticMethod(c, name, new Class[]{}, new Object[]{}); - } - - private Object invokeStaticMethod(Class c, String name, Class [] argTypes, Object [] params) { - try { - return c.getMethod(name, argTypes).invoke(null, params); - } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { - logger.error(e.getMessage(), e); - } - - return null; - } - - private Object instantiateClass(String name, Class [] argTypes, Object [] params) { - try { - Constructor constructor = getClass().getClassLoader() - .loadClass(name).getConstructor(argTypes); - return constructor.newInstance(params); - } catch (NoSuchMethodException | ClassNotFoundException | IllegalAccessException | - InstantiationException | InvocationTargetException e) { - logger.error(e.getMessage(), e); - } - return null; - } - - // function works after intp is initialized - boolean isScala2_10() { - try { - this.getClass().forName("org.apache.spark.repl.SparkIMain"); - return true; - } catch (ClassNotFoundException e) { - return false; - } - } - - boolean isScala2_11() { - return !isScala2_10(); - } private Class findClass(String name) { diff --git a/spark/src/main/java/org/apache/zeppelin/spark/Utils.java b/spark/src/main/java/org/apache/zeppelin/spark/Utils.java new file mode 100644 index 00000000000..94b06f485b4 --- /dev/null +++ b/spark/src/main/java/org/apache/zeppelin/spark/Utils.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +/** + * Utility and helper functions for the Spark Interpreter + */ +class Utils { + public static Logger logger = LoggerFactory.getLogger(Utils.class); + + static Object invokeMethod(Object o, String name) { + return invokeMethod(o, name, new Class[]{}, new Object[]{}); + } + + static Object invokeMethod(Object o, String name, Class[] argTypes, Object[] params) { + try { + return o.getClass().getMethod(name, argTypes).invoke(o, params); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + logger.error(e.getMessage(), e); + } + return null; + } + + static Object invokeStaticMethod(Class c, String name, Class[] argTypes, Object[] params) { + try { + return c.getMethod(name, argTypes).invoke(null, params); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + logger.error(e.getMessage(), e); + } + return null; + } + + static Object invokeStaticMethod(Class c, String name) { + return invokeStaticMethod(c, name, new Class[]{}, new Object[]{}); + } + + static Class findClass(String name) { + try { + return Utils.class.getClass().forName(name); + } catch (ClassNotFoundException e) { + logger.error(e.getMessage(), e); + return null; + } + } + + static Object instantiateClass(String name, Class[] argTypes, Object[] params) { + try { + Constructor constructor = Utils.class.getClass().getClassLoader() + .loadClass(name).getConstructor(argTypes); + return constructor.newInstance(params); + } catch (NoSuchMethodException | ClassNotFoundException | IllegalAccessException | + InstantiationException | InvocationTargetException e) { + logger.error(e.getMessage(), e); + } + return null; + } + + // function works after intp is initialized + static boolean isScala2_10() { + try { + Utils.class.getClass().forName("org.apache.spark.repl.SparkIMain"); + return true; + } catch (ClassNotFoundException e) { + return false; + } + } + + static boolean isScala2_11() { + return !isScala2_10(); + } +} From 87f46de8566481d4681fd93ce6405a8daab7075c Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Wed, 13 Jul 2016 08:12:18 -0700 Subject: [PATCH 29/31] Force build --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3b99b344ed6..2058cfa8090 100755 --- a/pom.xml +++ b/pom.xml @@ -389,7 +389,7 @@