From bdcfd1f894b70677399614e1588c16f524a0fab2 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 10 Feb 2017 09:44:04 +0900 Subject: [PATCH 1/4] Reduce log --- .../org/apache/zeppelin/interpreter/InterpreterSetting.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 74424303daf..cb0fe5225e6 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -145,8 +145,8 @@ private String getInterpreterProcessKey(String user, String noteId) { key = SHARED_PROCESS; } - logger.debug("getInterpreterProcessKey: {} for InterpreterSetting Id: {}, Name: {}", - key, getId(), getName()); + //logger.debug("getInterpreterProcessKey: {} for InterpreterSetting Id: {}, Name: {}", + // key, getId(), getName()); return key; } From 7860259ecf328c4dbcff480f501d557468930d7f Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 10 Feb 2017 09:44:12 +0900 Subject: [PATCH 2/4] reduce build time --- .travis.yml | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0e0ad693373..c59734003a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,21 +48,13 @@ matrix: - jdk: "oraclejdk7" env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS="" - # Test all modules with scala 2.10 + # Test spark module for 1.6.3 with scala 2.10 - jdk: "oraclejdk7" - env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Pbeam -Phelium-dev -Pexamples -Pscala-2.10" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS="" + env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.10" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" - # Test all modules with scala 2.11 + # Test spark module for 1.6.3 with scala 2.11 - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS="" - - # Test spark module for 1.5.2 - - jdk: "oraclejdk7" - env: SCALA_VER="2.10" SPARK_VER="1.5.2" HADOOP_VER="2.6" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.6 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false" - - # Test spark module for 1.4.1 - - jdk: "oraclejdk7" - env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.6 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false" + env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11 -Dscala.version=2.11.7 -Dscala.binary.version=2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk7" @@ -70,15 +62,15 @@ matrix: # Test python/pyspark with python 2 - jdk: "oraclejdk7" - env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" # Test python/pyspark with python 3 - jdk: "oraclejdk7" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python -Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -pl spark,python -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" # Test livy with spark 1.5.2 and hadoop 2.6 - jdk: "oraclejdk7" - env: SCALA_VER="2.10" $LIVY_VER="0.2.0" SPARK_VER="1.5.2" HADOOP_VER="2.6" PROFILE="-Pspark-1.5 -Phadoop-2.6" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,livy -DfailIfNoTests=false" + env: SCALA_VER="2.10" $LIVY_VER="0.2.0" SPARK_VER="1.5.2" HADOOP_VER="2.6" PROFILE="-Pspark-1.5 -Phadoop-2.6" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl zeppelin-interpreter,livy" TEST_PROJECTS="-DfailIfNoTests=false" before_install: - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc @@ -90,7 +82,7 @@ before_install: - source ~/.environ install: - - mvn $BUILD_FLAG $PROFILE -B + - mvn $BUILD_FLAG $MODULES $PROFILE -B before_script: - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER @@ -102,7 +94,7 @@ before_script: - tail conf/zeppelin-env.sh script: - - mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS + - mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS after_success: - echo "Travis exited with ${TRAVIS_TEST_RESULT}" From b14b97e637840dfb7dbd3a7b3a0cebf74076b0d0 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 10 Feb 2017 11:34:39 +0900 Subject: [PATCH 3/4] prevent concurrent test execution in DirAccessTest --- .../zeppelin/security/DirAccessTest.java | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/security/DirAccessTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/security/DirAccessTest.java index 820d0bad4d2..7837573ee3a 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/security/DirAccessTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/security/DirAccessTest.java @@ -28,24 +28,28 @@ public class DirAccessTest extends AbstractTestRestApi { @Test public void testDirAccessForbidden() throws Exception { - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED.getVarName(), "false"); - AbstractTestRestApi.startUpWithAuthenticationEnable(); - HttpClient httpClient = new HttpClient(); - GetMethod getMethod = new GetMethod(getUrlToTest() + "/app/"); - httpClient.executeMethod(getMethod); - AbstractTestRestApi.shutDown(); - assert getMethod.getStatusCode() == HttpStatus.SC_FORBIDDEN; + synchronized (this) { + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED.getVarName(), "false"); + AbstractTestRestApi.startUpWithAuthenticationEnable(); + HttpClient httpClient = new HttpClient(); + GetMethod getMethod = new GetMethod(getUrlToTest() + "/app/"); + httpClient.executeMethod(getMethod); + AbstractTestRestApi.shutDown(); + assert getMethod.getStatusCode() == HttpStatus.SC_FORBIDDEN; + } } @Test public void testDirAccessOk() throws Exception { - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED.getVarName(), "true"); - AbstractTestRestApi.startUpWithAuthenticationEnable(); - HttpClient httpClient = new HttpClient(); - GetMethod getMethod = new GetMethod(getUrlToTest() + "/app/"); - httpClient.executeMethod(getMethod); - AbstractTestRestApi.shutDown(); - assert getMethod.getStatusCode() == HttpStatus.SC_OK; + synchronized (this) { + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED.getVarName(), "true"); + AbstractTestRestApi.startUpWithAuthenticationEnable(); + HttpClient httpClient = new HttpClient(); + GetMethod getMethod = new GetMethod(getUrlToTest() + "/app/"); + httpClient.executeMethod(getMethod); + AbstractTestRestApi.shutDown(); + assert getMethod.getStatusCode() == HttpStatus.SC_OK; + } } protected static String getUrlToTest() { From 7e50ea524195278e245bb2348b59d8b9acaeb525 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 10 Feb 2017 11:36:29 +0900 Subject: [PATCH 4/4] create zeppelin-web/dist directory for test --- .../java/org/apache/zeppelin/rest/AbstractTestRestApi.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 2ff8d40b3de..fd65ae2bd77 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -127,6 +127,11 @@ private static void start(boolean withAuth) throws Exception { if (!wasRunning) { System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), "../"); System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_WAR.getVarName(), "../zeppelin-web/dist"); + + // some test profile does not build zeppelin-web. + // to prevent zeppelin starting up fail, create zeppelin-web/dist directory + new File("../zeppelin-web/dist").mkdirs(); + LOG.info("Staring test Zeppelin up..."); ZeppelinConfiguration conf = ZeppelinConfiguration.create();