[ZEPPELIN-759] Spark 2.0 support

.travis.yml

@@ -33,13 +33,17 @@ addons:
 
 matrix:
   include:
+    # Test all modules with spark-2.0.0-preview and scala 2.11
+    - jdk: "oraclejdk7"
+      env: SCALA_VER="2.11" SPARK_VER="2.0.0-preview" HADOOP_VER="2.3" PROFILE="-Pspark-2.0 -Dspark.version=2.0.0-preview -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
+
     # Test all modules with scala 2.10
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Dscala-2.10 -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
+      env: SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
 
     # Test all modules with scala 2.11
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Dscala-2.11 -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
+      env: SCALA_VER="2.11" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
 
     # Test spark module for 1.5.2
     - jdk: "oraclejdk7"
@@ -96,6 +100,8 @@ after_failure:
   - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
   - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
   - cat zeppelin-web/npm-debug.log
+  - cat spark-*/logs/*
 
 after_script:
   - ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER
+

README.md

@@ -124,6 +124,7 @@ Set spark major version
 Available profiles are
 
 ```
+-Pspark-2.0
 -Pspark-1.6
 -Pspark-1.5
 -Pspark-1.4
@@ -157,6 +158,16 @@ Available profiles are
 
 minor version can be adjusted by `-Dhadoop.version=x.x.x`
 
+##### `-Pscala-[version] (optional)`
+
+set scala version (default 2.10)
+Available profiles are
+
+```
+-Pscala-2.10
+-Pscala-2.11
+```
+
 ##### `-Pyarn` (optional)
 
 enable YARN support for local mode
@@ -199,14 +210,17 @@ Available profiles are
 Bulid examples under zeppelin-examples directory
 
 
-
+#### Example
 
 
 Here're some examples:
 
 ```sh
-# basic build
-mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark
+# build with spark-2.0, scala-2.11
+mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11
+
+# build with spark-1.6, scala-2.10
+mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr
 
 # spark-cassandra integration
 mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests

docs/install/install.md

@@ -93,8 +93,11 @@ mvn clean package -DskipTests [Options]
 Here are some examples with several options
 
 ```
-# basic build
-mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark
+# build with spark-2.0, scala-2.11
+mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11
+
+# build with spark-1.6, scala-2.10
+mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr
 
 # spark-cassandra integration
 mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests

r/pom.xml

@@ -383,7 +383,6 @@
         <property><name>!scala-2.11</name></property>
       </activation>
       <properties>
-        <spark.version>1.6.1</spark.version>
         <extra.source.dir>src/main/scala-2.10</extra.source.dir>
         <extra.testsource.dir>src/test/scala-2.10</extra.testsource.dir>
       </properties>
@@ -395,7 +394,6 @@
         <property><name>scala-2.11</name></property>
       </activation>
       <properties>
-        <spark.version>1.6.1</spark.version>
         <extra.source.dir>src/main/scala-2.11</extra.source.dir>
         <extra.testsource.dir>src/test/scala/scala-2.11</extra.testsource.dir>
       </properties>

spark-dependencies/pom.xml

@@ -36,8 +36,16 @@
   <url>http://zeppelin.apache.org</url>
 
   <properties>
-    <spark.version>1.4.1</spark.version>
+    <!-- library version defined in this section brought from spark 1.4.1 and it's dependency.
+         Therefore changing only spark.version is not going to be enough when this module
+         support new version of spark to make the new version as default supported version.
+
+         Each profile (spark-2.0, spark-1.6, etc) will overrides necessary dependency version.
+         So we'll make one of those profile 'activateByDefault' to make it default supported version
+         instead of changing spark.version in this section.
+    -->
 
+    <spark.version>1.4.1</spark.version>
     <hadoop.version>2.3.0</hadoop.version>
     <yarn.version>${hadoop.version}</yarn.version>
     <avro.version>1.7.7</avro.version>
@@ -283,12 +291,6 @@
       <version>${spark.version}</version>
     </dependency>
 
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-streaming-twitter_${scala.binary.version}</artifactId>
-      <version>${spark.version}</version>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
@@ -344,14 +346,6 @@
   </dependencies>
 
   <profiles>
-    <profile>
-      <id>scala-2.11</id>
-      <properties>
-        <spark.version>1.6.1</spark.version>
-        <spark.download.url>http://archive.apache.org/dist/spark/spark-${spark.version}/spark-${spark.version}.tgz</spark.download.url>
-      </properties>
-    </profile>
-
     <profile>
       <id>spark-1.1</id>
       <dependencies>
@@ -517,9 +511,6 @@
 
     <profile>
       <id>spark-1.6</id>
-      <activation>
-        <activeByDefault>true</activeByDefault>
-      </activation>
       <properties>
         <spark.version>1.6.1</spark.version>
         <py4j.version>0.9</py4j.version>
@@ -529,6 +520,19 @@
       </properties>
     </profile>
 
+    <profile>
+      <id>spark-2.0</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <properties>
+        <spark.version>2.0.0</spark.version>
+        <protobuf.version>2.5.0</protobuf.version>
+        <py4j.version>0.10.1</py4j.version>
+        <scala.version>2.11.8</scala.version>
+      </properties>
+    </profile>
+
     <profile>
       <id>hadoop-0.23</id>
       <!-- SPARK-1121: Adds an explicit dependency on Avro to work around a

spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java

@@ -528,6 +528,15 @@ public JavaSparkContext getJavaSparkContext() {
     }
   }
 
+  public Object getSparkSession() {
+    SparkInterpreter intp = getSparkInterpreter();
+    if (intp == null) {
+      return null;
+    } else {
+      return intp.getSparkSession();
+    }
+  }
+
   public SparkConf getSparkConf() {
     JavaSparkContext sc = getJavaSparkContext();
     if (sc == null) {