From 0303361b2c4390a837124afcfd5c231036665c36 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 27 Sep 2021 10:13:08 +0300 Subject: [PATCH 1/9] ZEPPELIN-5543 Add CircleCI config to test on Linux ARM64 --- .circleci/config.yml | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..3ebdb1232d7 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,70 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +version: 2.1 + +commands: + build-common: + steps: + - checkout + + - restore_cache: + keys: + - maven-repo-v1-{{ .Branch }}-{{ checksum "pom.xml" }} + - maven-repo-v1-{{ .Branch }}- + - maven-repo-v1- + - save_cache: + paths: + - ~/.m2 + key: maven-repo-v1-{{ .Branch }}-{{ checksum "pom.xml" }} + + - run: + name: Install build tools + command: | + sudo apt-get update -y + DEBIAN_FRONTEND=noninteractive \ + DEBCONF_NONINTERACTIVE_SEEN=true \ + TZ=UTC \ + sudo apt-get install -y --no-install-recommends openjdk-8-jdk maven + + - run: + name: Build + command: | + export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-arm64/" + export PATH="$JAVA_HOME/bin:$PATH" + java -version + mvn -version + mvn clean verify -Pspark-3.0 -Pspark-scala-2.12 -DskipTests --also-make-dependents --batch-mode --no-transfer-progress + +jobs: + build-arm64: + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.medium + + working_directory: ~/zeppelin + + steps: + - build-common + +workflows: + version: 2 + run-build-and-tests: + jobs: + - build-arm64 From ee2f0f8050fd057954f65e326f6a5af750c9cfcc Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 28 Sep 2021 09:41:06 +0300 Subject: [PATCH 2/9] ZEPPELIN-5543 Code review feedback - remove -DskipTests - delete ~/.m2/repository/org/apache/zeppelin after restoring the cache - always cache ~/.m2, not only on success. This should improve the speed - extract the JDK version as a parameter. This way it would be easy to add a new job for JDK 11, for example --- .circleci/config.yml | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ebdb1232d7..96dd13488c8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,6 +21,11 @@ version: 2.1 commands: build-common: + parameters: + jdk_version: + # 8, 11, 17, ... + type: integer + default: 8 steps: - checkout @@ -29,10 +34,10 @@ commands: - maven-repo-v1-{{ .Branch }}-{{ checksum "pom.xml" }} - maven-repo-v1-{{ .Branch }}- - maven-repo-v1- - - save_cache: - paths: - - ~/.m2 - key: maven-repo-v1-{{ .Branch }}-{{ checksum "pom.xml" }} + + - run: + name: Delete ~/.m2/repository/org/apache/zeppelin + command: rm -rf ~/.m2/repository/org/apache/zeppelin - run: name: Install build tools @@ -41,16 +46,22 @@ commands: DEBIAN_FRONTEND=noninteractive \ DEBCONF_NONINTERACTIVE_SEEN=true \ TZ=UTC \ - sudo apt-get install -y --no-install-recommends openjdk-8-jdk maven + sudo apt-get install -y --no-install-recommends openjdk-<< parameters.jdk_version >>-jdk maven - run: name: Build command: | - export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-arm64/" + export JAVA_HOME="/usr/lib/jvm/java-<< parameters.jdk_version >>-openjdk-arm64/" export PATH="$JAVA_HOME/bin:$PATH" java -version mvn -version - mvn clean verify -Pspark-3.0 -Pspark-scala-2.12 -DskipTests --also-make-dependents --batch-mode --no-transfer-progress + mvn clean verify -Pspark-3.0 -Pspark-scala-2.12 --also-make-dependents --batch-mode --no-transfer-progress + + - save_cache: + paths: + - ~/.m2 + key: maven-repo-v1-{{ .Branch }}-{{ checksum "pom.xml" }} + when: always jobs: build-arm64: From 77f692215371d849bfcb17ad97dac335389ad484 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 28 Sep 2021 10:18:22 +0300 Subject: [PATCH 3/9] ZEPPELIN-5543 Install wget It is needed by SparkInterpreterLauncherTest to download Apache Spark --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 96dd13488c8..80dda8f411c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,7 +46,7 @@ commands: DEBIAN_FRONTEND=noninteractive \ DEBCONF_NONINTERACTIVE_SEEN=true \ TZ=UTC \ - sudo apt-get install -y --no-install-recommends openjdk-<< parameters.jdk_version >>-jdk maven + sudo apt-get install -y --no-install-recommends openjdk-<< parameters.jdk_version >>-jdk maven wget - run: name: Build From 1fff1f5c35e3dfb3097682147d1d02b31fc893f9 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 28 Sep 2021 11:24:25 +0300 Subject: [PATCH 4/9] ZEPPELIN-5543 Split Maven build into install w/o tests + verify --- .circleci/config.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 80dda8f411c..9b397804c00 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,13 +49,20 @@ commands: sudo apt-get install -y --no-install-recommends openjdk-<< parameters.jdk_version >>-jdk maven wget - run: - name: Build + name: Use JDK << parameters.jdk_version >> command: | export JAVA_HOME="/usr/lib/jvm/java-<< parameters.jdk_version >>-openjdk-arm64/" export PATH="$JAVA_HOME/bin:$PATH" java -version mvn -version - mvn clean verify -Pspark-3.0 -Pspark-scala-2.12 --also-make-dependents --batch-mode --no-transfer-progress + + - run: + name: Maven install w/o tests + command: mvn clean install -DskipTests -Pspark-3.0 -Pspark-scala-2.12 --also-make-dependents --batch-mode --no-transfer-progress + + - run: + name: Maven verify + command: mvn clean verify -Pspark-3.0 -Pspark-scala-2.12 --also-make-dependents --batch-mode --no-transfer-progress - save_cache: paths: From 92ff9506610e488c1c342b9822046497f3a6af3e Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 28 Sep 2021 14:14:48 +0300 Subject: [PATCH 5/9] ZEPPELIN-5543 Do not fail 'mvn install' step There is some problem with the initial build of Zeppelin. 'mvn clean install' on fresh checkout does not build due to missing zeppelin-** dependencies from the Maven reactor --- .circleci/config.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b397804c00..9eaca9e277c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,17 +58,16 @@ commands: - run: name: Maven install w/o tests - command: mvn clean install -DskipTests -Pspark-3.0 -Pspark-scala-2.12 --also-make-dependents --batch-mode --no-transfer-progress - - - run: - name: Maven verify - command: mvn clean verify -Pspark-3.0 -Pspark-scala-2.12 --also-make-dependents --batch-mode --no-transfer-progress + command: mvn clean install -DskipTests -Pspark-3.0 -Pspark-scala-2.12 --also-make-dependents --batch-mode --no-transfer-progress --fail-never - save_cache: paths: - ~/.m2 key: maven-repo-v1-{{ .Branch }}-{{ checksum "pom.xml" }} - when: always + + - run: + name: Maven verify + command: mvn clean verify -Pspark-3.0 -Pspark-scala-2.12 --also-make-dependents --batch-mode --no-transfer-progress jobs: build-arm64: From 13e25b280cf8a6a4cdd38637aa020e599f3b49f7 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 28 Sep 2021 14:37:40 +0300 Subject: [PATCH 6/9] ZEPPELIN-5543 Make the setup similar to GitHub Actions 'core' workflow --- .circleci/config.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9eaca9e277c..a0bdb04ec4c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,8 +57,27 @@ commands: mvn -version - run: - name: Maven install w/o tests - command: mvn clean install -DskipTests -Pspark-3.0 -Pspark-scala-2.12 --also-make-dependents --batch-mode --no-transfer-progress --fail-never + name: Export environment variables + command: | + export MAVEN_OPTS="-Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3" + export ZEPPELIN_HELIUM_REGISTRY=helium + export SPARK_PRINT_LAUNCH_COMMAND="true" + export SPARK_LOCAL_IP="127.0.0.1" + export ZEPPELIN_LOCAL_IP="127.0.0.1" + + - run: + name: install application with some interpreter + command: | + mvn install -Pbuild-distr -DskipRat -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -Phadoop3 --batch-mode --no-transfer-progress --fail-never + + - run: + name: install and test plugins + command: mvn package -DskipRat -pl zeppelin-plugins -amd -B + + - run: + name: run tests with hadoop3 + command: | + mvn verify -Pusing-packaged-distr -DskipRat -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -Phadoop3 -Dtests.to.exclude=**/org/apache/zeppelin/spark/* -DfailIfNoTests=false - save_cache: paths: From f1014dfbdae5749e03661bc12866a32a4bae3075 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 28 Sep 2021 15:04:15 +0300 Subject: [PATCH 7/9] ZEPPELIN-5543 Exclude notebookrepo-mongo EmbeddedMongo does not support Linux ARM64 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0bdb04ec4c..26d38c66180 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,7 +72,7 @@ commands: - run: name: install and test plugins - command: mvn package -DskipRat -pl zeppelin-plugins -amd -B + command: mvn package -DskipRat -pl zeppelin-plugins,-org.apache.zeppelin:notebookrepo-mongo -amd -B - run: name: run tests with hadoop3 From b2dda9a79b74a3a66a066391ab68232f7ef1f752 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 28 Sep 2021 15:16:48 +0300 Subject: [PATCH 8/9] ZEPPELIN-5543 Properly setup the env vars for the whole build job --- .circleci/config.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 26d38c66180..f02a853ea60 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,6 +26,7 @@ commands: # 8, 11, 17, ... type: integer default: 8 + steps: - checkout @@ -49,22 +50,11 @@ commands: sudo apt-get install -y --no-install-recommends openjdk-<< parameters.jdk_version >>-jdk maven wget - run: - name: Use JDK << parameters.jdk_version >> + name: Print JDK and Maven versions command: | - export JAVA_HOME="/usr/lib/jvm/java-<< parameters.jdk_version >>-openjdk-arm64/" - export PATH="$JAVA_HOME/bin:$PATH" java -version mvn -version - - run: - name: Export environment variables - command: | - export MAVEN_OPTS="-Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3" - export ZEPPELIN_HELIUM_REGISTRY=helium - export SPARK_PRINT_LAUNCH_COMMAND="true" - export SPARK_LOCAL_IP="127.0.0.1" - export ZEPPELIN_LOCAL_IP="127.0.0.1" - - run: name: install application with some interpreter command: | @@ -94,6 +84,14 @@ jobs: image: ubuntu-2004:202101-01 resource_class: arm.medium + environment: + MAVEN_OPTS: "-Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3" + ZEPPELIN_HELIUM_REGISTRY: helium + SPARK_PRINT_LAUNCH_COMMAND: "true" + SPARK_LOCAL_IP: "127.0.0.1" + ZEPPELIN_LOCAL_IP: "127.0.0.1" + JAVA_HOME: "/usr/lib/jvm/java-8-openjdk-arm64/" + working_directory: ~/zeppelin steps: From 29d030f4f0ea64e37b1270958d2b5811b45e706b Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 29 Sep 2021 13:58:08 +0300 Subject: [PATCH 9/9] ZEPPELIN-5543 Try CirrusCI --- .cirrus.yml | 23 +++++++++++++++++++++++ Dockerfile | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 .cirrus.yml diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 00000000000..12f4b766180 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +test_task: + arm_container: + image: 3.8.2-openjdk-8-slim + test_script: mvn install -Pbuild-distr -DskipRat -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -Phadoop3 --batch-mode --no-transfer-progress --fail-never diff --git a/Dockerfile b/Dockerfile index 28606ea4cfb..3ba225eeded 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ WORKDIR /workspace/zeppelin RUN echo "unsafe-perm=true" > ~/.npmrc && \ echo '{ "allow_root": true }' > ~/.bowerrc && \ mvn -B package -DskipTests -Pbuild-distr -Pspark-3.0 -Pinclude-hadoop -Phadoop3 -Pspark-scala-2.12 -Pweb-angular && \ - # Example with doesn't compile all interpreters + # Example which doesn't compile all interpreters # mvn -B package -DskipTests -Pbuild-distr -Pspark-3.0 -Pinclude-hadoop -Phadoop3 -Pspark-scala-2.12 -Pweb-angular -pl '!groovy,!submarine,!livy,!hbase,!pig,!file,!flink,!ignite,!kylin,!lens' && \ mv /workspace/zeppelin/zeppelin-distribution/target/zeppelin-*/zeppelin-* /opt/zeppelin/ && \ # Removing stuff saves time, because docker creates a temporary layer