From e764f5fbb1b353d42c6f4dcdd1667d12942e5c38 Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Fri, 10 Jun 2016 13:35:42 -0700 Subject: [PATCH 01/12] Add maven publish release script --- dev/publish_release.sh | 149 +++++++++++++++++++++++++++++++++++++++++ pom.xml | 47 ++++++++----- 2 files changed, 178 insertions(+), 18 deletions(-) create mode 100755 dev/publish_release.sh diff --git a/dev/publish_release.sh b/dev/publish_release.sh new file mode 100755 index 00000000000..1ad9e432ed2 --- /dev/null +++ b/dev/publish_release.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The script helps publishing release to mavene. +# You need to specify release version and tag name. +# +# Here are some helpful documents for the release. +# http://www.apache.org/dev/release.html +# http://www.apache.org/dev/release-publishing +# http://www.apache.org/dev/release-signing.html +# http://www.apache.org/dev/publishing-maven-artifacts.html + +if [[ -z "${SHASUM}" ]]; then + SHASUM="/usr/bin/shasum -a 1" +fi + +if [[ -z "${WORKING_DIR}" ]]; then + WORKING_DIR=/tmp/zeppelin-release +fi + +if [ $# -ne 2 ]; then + echo "usage) $0 [Release version] [Tag]" + echo " ex. $0 0.6.0 v0.6.0" + exit 1 +fi + +for var in GPG_PASSPHRASE ASF_USERID ASF_PASSWORD +do + if [[ -z "${!var}" ]]; then + echo "You need ${var} variable set" + exit 1 + fi +done + +export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512m" + +RELEASE_VERSION="${1}" +GIT_TAG="${2}" + +PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" +PROJECT_OPTIONS="-pl !zeppelin-server,!zeppelin-distribution" +NEXUS_STAGING="https://repository.apache.org/service/local/staging" +NEXUS_PROFILE="153446d1ac37c4" + +if [[ -d "${WORKING_DIR}" ]]; then + echo "Dir ${WORKING_DIR} already exists" + exit 1 +fi + +mkdir ${WORKING_DIR} + +echo "Cloning the source" +# clone source +git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git ${WORKING_DIR}/zeppelin + +if [[ $? -ne 0 ]]; then + echo "Can not clone source repository" + exit 1 +fi + +cd ${WORKING_DIR}/zeppelin +git checkout ${GIT_TAG} +echo "Checked out ${GIT_TAG}" + +# remove unnecessary files +rm ${WORKING_DIR}/zeppelin/.gitignore +rm -rf ${WORKING_DIR}/zeppelin/.git + +function publish_to_maven() { + cd ${WORKING_DIR}/zeppelin + + # Force release version + mvn versions:set -DnewVersion=$RELEASE_VERSION + + # Using Nexus API documented here: + # https://support.sonatype.com/hc/en-us/articles/213465868-Uploading-to-a-Staging-Repository-via-REST-API + echo "Creating Nexus staging repository" + repo_request="Apache Zeppelin $RELEASE_VERSION" + out=$(curl -X POST -d "$repo_request" -u $ASF_USERID:$ASF_PASSWORD \ + -H "Content-Type:application/xml" -v \ + $NEXUS_STAGING/profiles/$NEXUS_PROFILE/start) + staged_repo_id=$(echo $out | sed -e "s/.*\(orgapachezeppelin-[0-9]\{4\}\).*/\1/") + echo "Created Nexus staging repository: $staged_repo_id" + + tmp_repo=$(mktemp -d /tmp/zeppelin-repo-XXXXX) + + echo "mvn clean install -Ppublish-distr \ + -Dmaven.repo.local=$tmp_repo \ + $PUBLISH_PROFILES $PROJECT_OPTIONS" + mvn clean install -Ppublish-distr -Dmaven.repo.local=$tmp_repo \ + $PUBLISH_PROFILES $PROJECT_OPTIONS + if [[ $? -ne 0 ]]; then + echo "Build failed." + exit 1 + fi + + pushd $tmp_repo/org/apache/zeppelin + find . -type f | grep -v \.jar$ | grep -v \.pom$ |grep -v \.war$ | xargs rm + + echo "Creating hash and signature files" + for file in $(find . -type f) + do + echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --output $file.asc \ + --detach-sig --armor $file; + md5 -q $file > $file.md5 + $SHASUM $file | cut -f1 -d' ' > $file.sha1 + done + + nexus_upload=$NEXUS_STAGING/deployByRepositoryId/$staged_repo_id + echo "Uplading files to $nexus_upload" + for file in $(find . -type f) + do + # strip leading ./ + file_short=$(echo $file | sed -e "s/\.\///") + dest_url="$nexus_upload/org/apache/zeppelin/$file_short" + echo " Uploading $file_short" + curl -u $ASF_USERID:$ASF_PASSWORD --upload-file $file_short $dest_url + done + + echo "Closing nexus staging repository" + repo_request="$staged_repo_idApache Zeppelin $RELEASE_VERSION" + out=$(curl -X POST -d "$repo_request" -u $ASF_USERID:$ASF_PASSWORD \ + -H "Content-Type:application/xml" -v \ + $NEXUS_STAGING/profiles/$NEXUS_PROFILE/finish) + echo "Closed Nexus staging repository: $staged_repo_id" + popd + rm -rf $tmp_repo +} + +publish_to_maven + +# remove working directory +rm -rf ${WORKING_DIR} diff --git a/pom.xml b/pom.xml index 0d4abc5e63c..a1e3e71234d 100755 --- a/pom.xml +++ b/pom.xml @@ -365,24 +365,6 @@ - - maven-javadoc-plugin - 2.9.1 - - - - - aggregate - - aggregate - - site - - - - - - maven-scm-plugin 1.8.1 @@ -724,6 +706,7 @@ false + @@ -732,6 +715,34 @@ true + + + org.apache.maven.plugins + maven-source-plugin + 3.0.0 + + + attach-sources + + jar + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.3 + + + attach-javadocs + + jar + + + + From cd1f08c46c0483e5c78cfe43445e598877595fb2 Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Fri, 10 Jun 2016 13:35:51 -0700 Subject: [PATCH 02/12] Refactor create release script --- dev/create_release.sh | 113 ++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 55 deletions(-) diff --git a/dev/create_release.sh b/dev/create_release.sh index b37fd514211..064d87aebf0 100755 --- a/dev/create_release.sh +++ b/dev/create_release.sh @@ -27,105 +27,108 @@ # http://www.apache.org/dev/publishing-maven-artifacts.html if [[ -z "${TAR}" ]]; then - TAR=/usr/bin/tar + TAR=/usr/bin/tar fi if [[ -z "${SHASUM}" ]]; then - SHASUM="/usr/bin/shasum -a 512" + SHASUM="/usr/bin/shasum -a 512" fi if [[ -z "${WORKING_DIR}" ]]; then - WORKING_DIR=/tmp/zeppelin-release + WORKING_DIR=/tmp/zeppelin-release fi if [[ -z "${GPG_PASSPHRASE}" ]]; then - echo "You need GPG_PASSPHRASE variable set" - exit 1 + echo "You need GPG_PASSPHRASE variable set" + exit 1 fi if [[ $# -ne 2 ]]; then - echo "usage) $0 [Release name] [Branch or Tag]" - echo " ex. $0 0.6.0 branch-0.6" - exit 1 + echo "usage) $0 [Release version] [Tag]" + echo " ex. $0 0.6.0 v0.6.0-rc1" + exit 1 fi -RELEASE_NAME="${1}" -BRANCH="${2}" - +RELEASE_VERSION="${1}" +GIT_TAG="${2}" if [[ -d "${WORKING_DIR}" ]]; then - echo "Dir ${WORKING_DIR} already exists" - exit 1 + echo "Dir ${WORKING_DIR} already exists" + exit 1 fi mkdir ${WORKING_DIR} echo "Cloning the source and packaging" # clone source -git clone -b ${BRANCH} git@github.com:apache/zeppelin.git ${WORKING_DIR}/zeppelin +git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git ${WORKING_DIR}/zeppelin + if [[ $? -ne 0 ]]; then - echo "Can not clone source repository" - exit 1 + echo "Can not clone source repository" + exit 1 fi +cd ${WORKING_DIR}/zeppelin +git checkout ${GIT_TAG} +echo "Checked out ${GIT_TAG}" + # remove unnecessary files rm ${WORKING_DIR}/zeppelin/.gitignore rm -rf ${WORKING_DIR}/zeppelin/.git - # create source package cd ${WORKING_DIR} -cp -r zeppelin zeppelin-${RELEASE_NAME} -${TAR} cvzf zeppelin-${RELEASE_NAME}.tgz zeppelin-${RELEASE_NAME} +cp -r zeppelin zeppelin-${RELEASE_VERSION} +${TAR} cvzf zeppelin-${RELEASE_VERSION}.tgz zeppelin-${RELEASE_VERSION} echo "Signing the source package" cd ${WORKING_DIR} -echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --armor --output zeppelin-${RELEASE_NAME}.tgz.asc --detach-sig ${WORKING_DIR}/zeppelin-${RELEASE_NAME}.tgz -echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --print-md MD5 zeppelin-${RELEASE_NAME}.tgz > ${WORKING_DIR}/zeppelin-${RELEASE_NAME}.tgz.md5 -${SHASUM} zeppelin-${RELEASE_NAME}.tgz > ${WORKING_DIR}/zeppelin-${RELEASE_NAME}.tgz.sha512 +echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --armor --output zeppelin-${RELEASE_VERSION}.tgz.asc --detach-sig ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz +echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --print-md MD5 zeppelin-${RELEASE_VERSION}.tgz > ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz.md5 +${SHASUM} zeppelin-${RELEASE_VERSION}.tgz > ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz.sha512 function make_binary_release() { - BIN_RELEASE_NAME="${1}" - BUILD_FLAGS="${2}" - - cp -r ${WORKING_DIR}/zeppelin ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME} - cd ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME} - echo "mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}" - mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS} - if [[ $? -ne 0 ]]; then - echo "Build failed. ${BUILD_FLAGS}" - exit 1 - fi - - # re-create package with proper dir name with binary license - cd zeppelin-distribution/target/zeppelin-* - mv zeppelin-* zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME} - cat ../../src/bin_license/LICENSE >> zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}/LICENSE - cat ../../src/bin_license/NOTICE >> zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}/NOTICE - cp ../../src/bin_license/licenses/* zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}/licenses/ - ${TAR} cvzf zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME} - - # sign bin package - echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --armor --output zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.asc --detach-sig zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz - echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --print-md MD5 zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz > zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.md5 - ${SHASUM} zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz > zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.sha512 - - mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz ${WORKING_DIR}/ - mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.asc ${WORKING_DIR}/ - mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.md5 ${WORKING_DIR}/ - mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.sha512 ${WORKING_DIR}/ - - # clean up build dir - rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME} + BIN_RELEASE_NAME="${1}" + BUILD_FLAGS="${2}" + + cp -r ${WORKING_DIR}/zeppelin ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} + cd ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} + echo "mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}" + mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS} + if [[ $? -ne 0 ]]; then + echo "Build failed. ${BUILD_FLAGS}" + exit 1 + fi + + # re-create package with proper dir name with binary license + cd zeppelin-distribution/target/zeppelin-* + mv zeppelin-* zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} + cat ../../src/bin_license/LICENSE >> zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/LICENSE + cat ../../src/bin_license/NOTICE >> zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/NOTICE + cp ../../src/bin_license/licenses/* zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/licenses/ + ${TAR} cvzf zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} + + # sign bin package + echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --armor --output zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.asc --detach-sig zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz + echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --print-md MD5 zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz > zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.md5 + ${SHASUM} zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz > zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.sha512 + + mv zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz ${WORKING_DIR}/ + mv zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.asc ${WORKING_DIR}/ + mv zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.md5 ${WORKING_DIR}/ + mv zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.sha512 ${WORKING_DIR}/ + + # clean up build dir + rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} } make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark" # remove non release files and dirs rm -rf ${WORKING_DIR}/zeppelin -rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_NAME} +rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_VERSION} echo "Release files are created under ${WORKING_DIR}" From 30dcc65b82cdabd0c9df2d1c4f4adf16ece0e11c Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Fri, 10 Jun 2016 14:45:16 -0700 Subject: [PATCH 03/12] remove deploy plugin from pom since custom script will be used instead for deploy --- alluxio/pom.xml | 9 --------- angular/pom.xml | 9 --------- cassandra/pom.xml | 9 --------- elasticsearch/pom.xml | 9 --------- file/pom.xml | 9 --------- flink/pom.xml | 9 --------- geode/pom.xml | 9 --------- hbase/pom.xml | 9 --------- ignite/pom.xml | 9 --------- jdbc/pom.xml | 9 --------- kylin/pom.xml | 9 --------- lens/pom.xml | 9 --------- livy/pom.xml | 9 --------- markdown/pom.xml | 9 --------- postgresql/pom.xml | 9 --------- python/pom.xml | 9 --------- r/pom.xml | 9 --------- scalding/pom.xml | 9 --------- shell/pom.xml | 9 --------- spark-dependencies/pom.xml | 9 --------- zeppelin-display/pom.xml | 9 --------- zeppelin-distribution/pom.xml | 18 ------------------ zeppelin-server/pom.xml | 9 --------- 23 files changed, 216 deletions(-) diff --git a/alluxio/pom.xml b/alluxio/pom.xml index fa5df776778..bee0091f11e 100644 --- a/alluxio/pom.xml +++ b/alluxio/pom.xml @@ -132,15 +132,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/angular/pom.xml b/angular/pom.xml index 9e732b018ae..b6251f133c9 100644 --- a/angular/pom.xml +++ b/angular/pom.xml @@ -60,15 +60,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/cassandra/pom.xml b/cassandra/pom.xml index 46d2530d462..df00522e041 100644 --- a/cassandra/pom.xml +++ b/cassandra/pom.xml @@ -230,15 +230,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml index 7f53299957e..b26dad6af05 100644 --- a/elasticsearch/pom.xml +++ b/elasticsearch/pom.xml @@ -79,15 +79,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/file/pom.xml b/file/pom.xml index 2410432186f..6eb4a92577f 100644 --- a/file/pom.xml +++ b/file/pom.xml @@ -72,15 +72,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - org.apache.maven.plugins maven-surefire-plugin diff --git a/flink/pom.xml b/flink/pom.xml index 16187a1795d..7d096a34263 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -294,15 +294,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/geode/pom.xml b/geode/pom.xml index 961483c9c9c..9e49efe1dbf 100644 --- a/geode/pom.xml +++ b/geode/pom.xml @@ -84,15 +84,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/hbase/pom.xml b/hbase/pom.xml index 3b8fcb6f8fe..54960257292 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -109,15 +109,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/ignite/pom.xml b/ignite/pom.xml index 260ed29f937..d441c462a0d 100644 --- a/ignite/pom.xml +++ b/ignite/pom.xml @@ -107,15 +107,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 550f439a932..aab9e3a346d 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -102,15 +102,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/kylin/pom.xml b/kylin/pom.xml index 34ef235c241..f9623186eb2 100755 --- a/kylin/pom.xml +++ b/kylin/pom.xml @@ -61,15 +61,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/lens/pom.xml b/lens/pom.xml index 667475fc6ab..1c07ecad8c9 100644 --- a/lens/pom.xml +++ b/lens/pom.xml @@ -146,15 +146,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/livy/pom.xml b/livy/pom.xml index f739efdda79..90f149ca1cb 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -100,15 +100,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/markdown/pom.xml b/markdown/pom.xml index 1c5e174b31c..506e41fb829 100644 --- a/markdown/pom.xml +++ b/markdown/pom.xml @@ -66,15 +66,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/postgresql/pom.xml b/postgresql/pom.xml index f5554fa8a2d..641a6503b9d 100644 --- a/postgresql/pom.xml +++ b/postgresql/pom.xml @@ -95,15 +95,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/python/pom.xml b/python/pom.xml index 8db5feda676..b91142e24e1 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -87,15 +87,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/r/pom.xml b/r/pom.xml index 0684bb10f43..a6bec5284dc 100644 --- a/r/pom.xml +++ b/r/pom.xml @@ -205,15 +205,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/scalding/pom.xml b/scalding/pom.xml index a3b3b58fe63..7dbaf76da07 100644 --- a/scalding/pom.xml +++ b/scalding/pom.xml @@ -142,15 +142,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/shell/pom.xml b/shell/pom.xml index 413b4a60ef3..2f73ea63d54 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -66,15 +66,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml index 143e9cc549c..556f563570e 100644 --- a/spark-dependencies/pom.xml +++ b/spark-dependencies/pom.xml @@ -970,15 +970,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - maven-enforcer-plugin 1.3.1 diff --git a/zeppelin-display/pom.xml b/zeppelin-display/pom.xml index e648d6a089c..b112b8b0e36 100644 --- a/zeppelin-display/pom.xml +++ b/zeppelin-display/pom.xml @@ -132,15 +132,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - org.scala-tools maven-scala-plugin diff --git a/zeppelin-distribution/pom.xml b/zeppelin-distribution/pom.xml index d7beb490d95..10973938897 100644 --- a/zeppelin-distribution/pom.xml +++ b/zeppelin-distribution/pom.xml @@ -72,24 +72,6 @@ - - - - maven-dependency-plugin - - true - - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 675a2ce0148..270eb5839d2 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -327,15 +327,6 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - - org.scala-tools maven-scala-plugin From aafaf4210afb0b55547c75fdaa1d07c10a99f8cc Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Tue, 14 Jun 2016 13:45:23 -0700 Subject: [PATCH 04/12] Follow google shell style guide --- dev/publish_release.sh | 91 ++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 47 deletions(-) diff --git a/dev/publish_release.sh b/dev/publish_release.sh index 1ad9e432ed2..e3448dccd14 100755 --- a/dev/publish_release.sh +++ b/dev/publish_release.sh @@ -31,7 +31,7 @@ if [[ -z "${SHASUM}" ]]; then fi if [[ -z "${WORKING_DIR}" ]]; then - WORKING_DIR=/tmp/zeppelin-release + WORKING_DIR="/tmp/zeppelin-release" fi if [ $# -ne 2 ]; then @@ -40,8 +40,7 @@ if [ $# -ne 2 ]; then exit 1 fi -for var in GPG_PASSPHRASE ASF_USERID ASF_PASSWORD -do +for var in GPG_PASSPHRASE; do ASF_USERID ASF_PASSWORD; do if [[ -z "${!var}" ]]; then echo "You need ${var} variable set" exit 1 @@ -50,8 +49,8 @@ done export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512m" -RELEASE_VERSION="${1}" -GIT_TAG="${2}" +RELEASE_VERSION="$1" +GIT_TAG="$2" PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" PROJECT_OPTIONS="-pl !zeppelin-server,!zeppelin-distribution" @@ -63,87 +62,85 @@ if [[ -d "${WORKING_DIR}" ]]; then exit 1 fi -mkdir ${WORKING_DIR} +mkdir "${WORKING_DIR}" echo "Cloning the source" # clone source -git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git ${WORKING_DIR}/zeppelin +git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git "${WORKING_DIR}/zeppelin" if [[ $? -ne 0 ]]; then echo "Can not clone source repository" exit 1 fi -cd ${WORKING_DIR}/zeppelin -git checkout ${GIT_TAG} +cd "${WORKING_DIR}/zeppelin" +git checkout "${GIT_TAG}" echo "Checked out ${GIT_TAG}" # remove unnecessary files -rm ${WORKING_DIR}/zeppelin/.gitignore -rm -rf ${WORKING_DIR}/zeppelin/.git +rm "${WORKING_DIR}/zeppelin/.gitignore" +rm -rf "${WORKING_DIR}/zeppelin/.git" function publish_to_maven() { - cd ${WORKING_DIR}/zeppelin + cd "${WORKING_DIR}/zeppelin" # Force release version - mvn versions:set -DnewVersion=$RELEASE_VERSION + mvn versions:set -DnewVersion="${RELEASE_VERSION}" # Using Nexus API documented here: # https://support.sonatype.com/hc/en-us/articles/213465868-Uploading-to-a-Staging-Repository-via-REST-API echo "Creating Nexus staging repository" - repo_request="Apache Zeppelin $RELEASE_VERSION" - out=$(curl -X POST -d "$repo_request" -u $ASF_USERID:$ASF_PASSWORD \ - -H "Content-Type:application/xml" -v \ - $NEXUS_STAGING/profiles/$NEXUS_PROFILE/start) - staged_repo_id=$(echo $out | sed -e "s/.*\(orgapachezeppelin-[0-9]\{4\}\).*/\1/") - echo "Created Nexus staging repository: $staged_repo_id" + repo_request="Apache Zeppelin ${RELEASE_VERSION}" + out="$(curl -X POST -d "${repo_request}" -u "${ASF_USERID}:${ASF_PASSWORD}" \ + -H 'Content-Type:application/xml' -v \ + "${NEXUS_STAGING}/profiles/${NEXUS_PROFILE}/start")" + staged_repo_id="$(echo "${out}" | sed -e 's/.*\(orgapachezeppelin-[0-9]\{4\}\).*/\1/')" + echo "Created Nexus staging repository: ${staged_repo_id}" - tmp_repo=$(mktemp -d /tmp/zeppelin-repo-XXXXX) + tmp_repo="$(mktemp -d /tmp/zeppelin-repo-XXXXX)" echo "mvn clean install -Ppublish-distr \ - -Dmaven.repo.local=$tmp_repo \ - $PUBLISH_PROFILES $PROJECT_OPTIONS" - mvn clean install -Ppublish-distr -Dmaven.repo.local=$tmp_repo \ - $PUBLISH_PROFILES $PROJECT_OPTIONS + -Dmaven.repo.local=${tmp_repo} \ + ${PUBLISH_PROFILES} ${PROJECT_OPTIONS}" + mvn clean install -Ppublish-distr -Dmaven.repo.local="${tmp_repo}" \ + ${PUBLISH_PROFILES} ${PROJECT_OPTIONS} if [[ $? -ne 0 ]]; then echo "Build failed." exit 1 fi - pushd $tmp_repo/org/apache/zeppelin - find . -type f | grep -v \.jar$ | grep -v \.pom$ |grep -v \.war$ | xargs rm + pushd "${tmp_repo}/org/apache/zeppelin" + find . -type f | grep -v '\.jar$' | grep -v '\.pom$' |grep -v '\.war$' | xargs rm echo "Creating hash and signature files" - for file in $(find . -type f) - do - echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --output $file.asc \ - --detach-sig --armor $file; - md5 -q $file > $file.md5 - $SHASUM $file | cut -f1 -d' ' > $file.sha1 + for file in $(find . -type f); do + echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --output "${file}.asc" \ + --detach-sig --armor "${file}" + md5 -q "${file}" > "${file}.md5" + ${SHASUM} "${file}" | cut -f1 -d' ' > "${file}.sha1" done - nexus_upload=$NEXUS_STAGING/deployByRepositoryId/$staged_repo_id - echo "Uplading files to $nexus_upload" - for file in $(find . -type f) - do + nexus_upload="${NEXUS_STAGING}/deployByRepositoryId/${staged_repo_id}" + echo "Uplading files to ${nexus_upload}" + for file in $(find . -type f); do # strip leading ./ - file_short=$(echo $file | sed -e "s/\.\///") - dest_url="$nexus_upload/org/apache/zeppelin/$file_short" - echo " Uploading $file_short" - curl -u $ASF_USERID:$ASF_PASSWORD --upload-file $file_short $dest_url + file_short="$(echo "${file}" | sed -e 's/\.\///')" + dest_url="${nexus_upload}/org/apache/zeppelin/$file_short" + echo " Uploading ${file_short}" + curl -u "${ASF_USERID}:${ASF_PASSWORD}" --upload-file "${file_short}" "${dest_url}" done echo "Closing nexus staging repository" - repo_request="$staged_repo_idApache Zeppelin $RELEASE_VERSION" - out=$(curl -X POST -d "$repo_request" -u $ASF_USERID:$ASF_PASSWORD \ - -H "Content-Type:application/xml" -v \ - $NEXUS_STAGING/profiles/$NEXUS_PROFILE/finish) - echo "Closed Nexus staging repository: $staged_repo_id" + repo_request="${staged_repo_id}Apache Zeppelin ${RELEASE_VERSION}" + out="$(curl -X POST -d "${repo_request}" -u "${ASF_USERID}:${ASF_PASSWORD}" \ + -H 'Content-Type:application/xml' -v \ + "${NEXUS_STAGING}}/profiles/${NEXUS_PROFILE}/finish")" + echo "Closed Nexus staging repository: ${staged_repo_id}" popd - rm -rf $tmp_repo + rm -rf "${tmp_repo}" } publish_to_maven # remove working directory -rm -rf ${WORKING_DIR} +rm -rf "${WORKING_DIR}" From 48d338f1f76e1b6307b899c4f40dcac1e123d02d Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Wed, 15 Jun 2016 10:08:00 -0700 Subject: [PATCH 05/12] Rollback mistakenly removed plugin --- zeppelin-distribution/pom.xml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zeppelin-distribution/pom.xml b/zeppelin-distribution/pom.xml index 10973938897..9bc4ac152eb 100644 --- a/zeppelin-distribution/pom.xml +++ b/zeppelin-distribution/pom.xml @@ -72,8 +72,15 @@ - + + + maven-dependency-plugin + + true + + + From a3eb67673c30e10efe181810c554729b5d8250e9 Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Wed, 15 Jun 2016 16:01:23 -0700 Subject: [PATCH 06/12] Include zeppelin-server module in publish artifiact --- dev/publish_release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/publish_release.sh b/dev/publish_release.sh index e3448dccd14..3bd3cca8ee3 100755 --- a/dev/publish_release.sh +++ b/dev/publish_release.sh @@ -53,7 +53,7 @@ RELEASE_VERSION="$1" GIT_TAG="$2" PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" -PROJECT_OPTIONS="-pl !zeppelin-server,!zeppelin-distribution" +PROJECT_OPTIONS="-pl !zeppelin-distribution" NEXUS_STAGING="https://repository.apache.org/service/local/staging" NEXUS_PROFILE="153446d1ac37c4" From 903bc125118eeb1520cb1deb664cab90353acb40 Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Thu, 16 Jun 2016 18:17:37 -0700 Subject: [PATCH 07/12] Move duplicate code to common_release.sh --- dev/common_release.sh | 61 +++++++++++++++++++ dev/create_release.sh | 134 +++++++++++++++++------------------------ dev/publish_release.sh | 57 ++++-------------- 3 files changed, 128 insertions(+), 124 deletions(-) create mode 100644 dev/common_release.sh diff --git a/dev/common_release.sh b/dev/common_release.sh new file mode 100644 index 00000000000..df9c33db00f --- /dev/null +++ b/dev/common_release.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# common fucntions + +if [[ -z "${TAR}" ]]; then + TAR="/usr/bin/tar" +fi + +if [[ -z "${SHASUM}" ]]; then + SHASUM="/usr/bin/shasum" +fi + +if [[ -z "${WORKING_DIR}" ]]; then + WORKING_DIR="/tmp/zeppelin-release" +fi + +mkdir "${WORKING_DIR}" + +usage() { + echo "usage) $0 [Release version] [Branch or Tag]" + echo " ex. $0 0.6.0 v0.6.0" + exit 1 +} + +function git_clone() { + echo "Clone the source" + # clone source + git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git "${WORKING_DIR}/zeppelin" + + if [[ $? -ne 0 ]]; then + echo "Can not clone source repository" + exit 1 + fi + + cd "${WORKING_DIR}/zeppelin" + git checkout "${GIT_TAG}" + echo "Checked out ${GIT_TAG}" + + # remove unnecessary files + rm "${WORKING_DIR}/zeppelin/.gitignore" + rm -rf "${WORKING_DIR}/zeppelin/.git" + rm -rf "${WORKING_DIR}/zeppelin/.github" + rm -rf "${WORKING_DIR}/zeppelin/docs" +} diff --git a/dev/create_release.sh b/dev/create_release.sh index 064d87aebf0..ceccbf316bf 100755 --- a/dev/create_release.sh +++ b/dev/create_release.sh @@ -18,25 +18,19 @@ # # The script helps making a release. -# You need specify a release name and branch|tag name. +# You need to specify release version and branch|tag name. # -# Here's some helpful documents for the release +# Here are some helpful documents for the release. # http://www.apache.org/dev/release.html # http://www.apache.org/dev/release-publishing # http://www.apache.org/dev/release-signing.html -# http://www.apache.org/dev/publishing-maven-artifacts.html - -if [[ -z "${TAR}" ]]; then - TAR=/usr/bin/tar -fi - -if [[ -z "${SHASUM}" ]]; then - SHASUM="/usr/bin/shasum -a 512" -fi +BASEDIR="$(dirname "$0")" +. "${BASEDIR}/common_release.sh" +echo "${BASEDIR}/common_release.sh" -if [[ -z "${WORKING_DIR}" ]]; then - WORKING_DIR=/tmp/zeppelin-release +if [[ $# -ne 2 ]]; then + usage fi if [[ -z "${GPG_PASSPHRASE}" ]]; then @@ -44,59 +38,34 @@ if [[ -z "${GPG_PASSPHRASE}" ]]; then exit 1 fi - -if [[ $# -ne 2 ]]; then - echo "usage) $0 [Release version] [Tag]" - echo " ex. $0 0.6.0 v0.6.0-rc1" - exit 1 -fi - -RELEASE_VERSION="${1}" -GIT_TAG="${2}" - -if [[ -d "${WORKING_DIR}" ]]; then - echo "Dir ${WORKING_DIR} already exists" - exit 1 -fi - -mkdir ${WORKING_DIR} - -echo "Cloning the source and packaging" -# clone source -git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git ${WORKING_DIR}/zeppelin - -if [[ $? -ne 0 ]]; then - echo "Can not clone source repository" - exit 1 -fi - -cd ${WORKING_DIR}/zeppelin -git checkout ${GIT_TAG} -echo "Checked out ${GIT_TAG}" - -# remove unnecessary files -rm ${WORKING_DIR}/zeppelin/.gitignore -rm -rf ${WORKING_DIR}/zeppelin/.git - - -# create source package -cd ${WORKING_DIR} -cp -r zeppelin zeppelin-${RELEASE_VERSION} -${TAR} cvzf zeppelin-${RELEASE_VERSION}.tgz zeppelin-${RELEASE_VERSION} - -echo "Signing the source package" -cd ${WORKING_DIR} -echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --armor --output zeppelin-${RELEASE_VERSION}.tgz.asc --detach-sig ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz -echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --print-md MD5 zeppelin-${RELEASE_VERSION}.tgz > ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz.md5 -${SHASUM} zeppelin-${RELEASE_VERSION}.tgz > ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz.sha512 - +RELEASE_VERSION="$1" +GIT_TAG="$2" + +function make_source_package() { + # create source package + cd ${WORKING_DIR} + cp -r "zeppelin" "zeppelin-${RELEASE_VERSION}" + ${TAR} cvzf "zeppelin-${RELEASE_VERSION}.tgz" "zeppelin-${RELEASE_VERSION}" + + echo "Signing the source package" + cd "${WORKING_DIR}" + echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --armor \ + --output "zeppelin-${RELEASE_VERSION}.tgz.asc" \ + --detach-sig "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz" + echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 \ + --print-md MD5 "zeppelin-${RELEASE_VERSION}.tgz" > \ + "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz.md5" + echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 \ + --print-md SHA512 "zeppelin-${RELEASE_VERSION}.tgz" > \ + "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}.tgz.sha512" +} function make_binary_release() { - BIN_RELEASE_NAME="${1}" - BUILD_FLAGS="${2}" + BIN_RELEASE_NAME="$1" + BUILD_FLAGS="$2" - cp -r ${WORKING_DIR}/zeppelin ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} - cd ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} + cp -r "${WORKING_DIR}/zeppelin" "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}" + cd "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}" echo "mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}" mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS} if [[ $? -ne 0 ]]; then @@ -106,29 +75,36 @@ function make_binary_release() { # re-create package with proper dir name with binary license cd zeppelin-distribution/target/zeppelin-* - mv zeppelin-* zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} - cat ../../src/bin_license/LICENSE >> zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/LICENSE - cat ../../src/bin_license/NOTICE >> zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/NOTICE - cp ../../src/bin_license/licenses/* zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/licenses/ - ${TAR} cvzf zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} + mv zeppelin-* "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}" + cat ../../src/bin_license/LICENSE >> "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/LICENSE" + cat ../../src/bin_license/NOTICE >> "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/NOTICE" + cp ../../src/bin_license/licenses/* "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}/licenses/" + ${TAR} cvzf "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz" "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}" # sign bin package - echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --armor --output zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.asc --detach-sig zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz - echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --print-md MD5 zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz > zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.md5 - ${SHASUM} zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz > zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.sha512 - - mv zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz ${WORKING_DIR}/ - mv zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.asc ${WORKING_DIR}/ - mv zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.md5 ${WORKING_DIR}/ - mv zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.sha512 ${WORKING_DIR}/ + echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --armor \ + --output "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.asc" \ + --detach-sig "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz" + echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --print-md MD5 \ + "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz" > \ + "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.md5" + ${SHASUM} -a 512 "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz" > \ + "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.sha512" + + mv "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz" "${WORKING_DIR}/" + mv "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.asc" "${WORKING_DIR}/" + mv "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.md5" "${WORKING_DIR}/" + mv "zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}.tgz.sha512" "${WORKING_DIR}/" # clean up build dir - rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME} + rm -rf "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}" } +git_clone +make_source_package make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark" # remove non release files and dirs -rm -rf ${WORKING_DIR}/zeppelin -rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_VERSION} +rm -rf "${WORKING_DIR}/zeppelin" +rm -rf "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}" echo "Release files are created under ${WORKING_DIR}" diff --git a/dev/publish_release.sh b/dev/publish_release.sh index 3bd3cca8ee3..b5625ba855b 100755 --- a/dev/publish_release.sh +++ b/dev/publish_release.sh @@ -17,30 +17,20 @@ # limitations under the License. # -# The script helps publishing release to mavene. -# You need to specify release version and tag name. +# The script helps publishing release to maven. +# You need to specify release version and branch|tag name. # -# Here are some helpful documents for the release. -# http://www.apache.org/dev/release.html -# http://www.apache.org/dev/release-publishing -# http://www.apache.org/dev/release-signing.html +# Here's some helpful documents for the release. # http://www.apache.org/dev/publishing-maven-artifacts.html -if [[ -z "${SHASUM}" ]]; then - SHASUM="/usr/bin/shasum -a 1" -fi - -if [[ -z "${WORKING_DIR}" ]]; then - WORKING_DIR="/tmp/zeppelin-release" -fi +BASEDIR="$(dirname "$0")" +. "${BASEDIR}/common_release.sh" -if [ $# -ne 2 ]; then - echo "usage) $0 [Release version] [Tag]" - echo " ex. $0 0.6.0 v0.6.0" - exit 1 +if [[ $# -ne 2 ]]; then + usage fi -for var in GPG_PASSPHRASE; do ASF_USERID ASF_PASSWORD; do +for var in GPG_PASSPHRASE ASF_USERID ASF_PASSWORD; do if [[ -z "${!var}" ]]; then echo "You need ${var} variable set" exit 1 @@ -52,35 +42,11 @@ export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512m" RELEASE_VERSION="$1" GIT_TAG="$2" -PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" +PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark" PROJECT_OPTIONS="-pl !zeppelin-distribution" NEXUS_STAGING="https://repository.apache.org/service/local/staging" NEXUS_PROFILE="153446d1ac37c4" -if [[ -d "${WORKING_DIR}" ]]; then - echo "Dir ${WORKING_DIR} already exists" - exit 1 -fi - -mkdir "${WORKING_DIR}" - -echo "Cloning the source" -# clone source -git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git "${WORKING_DIR}/zeppelin" - -if [[ $? -ne 0 ]]; then - echo "Can not clone source repository" - exit 1 -fi - -cd "${WORKING_DIR}/zeppelin" -git checkout "${GIT_TAG}" -echo "Checked out ${GIT_TAG}" - -# remove unnecessary files -rm "${WORKING_DIR}/zeppelin/.gitignore" -rm -rf "${WORKING_DIR}/zeppelin/.git" - function publish_to_maven() { cd "${WORKING_DIR}/zeppelin" @@ -117,7 +83,7 @@ function publish_to_maven() { echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --output "${file}.asc" \ --detach-sig --armor "${file}" md5 -q "${file}" > "${file}.md5" - ${SHASUM} "${file}" | cut -f1 -d' ' > "${file}.sha1" + ${SHASUM} -a 1 "${file}" | cut -f1 -d' ' > "${file}.sha1" done nexus_upload="${NEXUS_STAGING}/deployByRepositoryId/${staged_repo_id}" @@ -140,7 +106,8 @@ function publish_to_maven() { rm -rf "${tmp_repo}" } +git_clone publish_to_maven # remove working directory -rm -rf "${WORKING_DIR}" +rm -rf "${WORKING_DIR}/zeppelin" From 2cced613575082544e4982e0c8d86da9143e79d7 Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Sun, 19 Jun 2016 23:39:45 -0700 Subject: [PATCH 08/12] Add r to binary package and maven build --- dev/create_release.sh | 2 +- dev/publish_release.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/create_release.sh b/dev/create_release.sh index ceccbf316bf..f1bba0dded6 100755 --- a/dev/create_release.sh +++ b/dev/create_release.sh @@ -102,7 +102,7 @@ function make_binary_release() { git_clone make_source_package -make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark" +make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" # remove non release files and dirs rm -rf "${WORKING_DIR}/zeppelin" diff --git a/dev/publish_release.sh b/dev/publish_release.sh index b5625ba855b..04cc061974e 100755 --- a/dev/publish_release.sh +++ b/dev/publish_release.sh @@ -42,7 +42,7 @@ export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512m" RELEASE_VERSION="$1" GIT_TAG="$2" -PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark" +PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" PROJECT_OPTIONS="-pl !zeppelin-distribution" NEXUS_STAGING="https://repository.apache.org/service/local/staging" NEXUS_PROFILE="153446d1ac37c4" From a88bc1dd2b200e06b32654c7a0654b11ea0421df Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Sun, 19 Jun 2016 23:40:41 -0700 Subject: [PATCH 09/12] Add geode, scalding profile in maven artifact build --- dev/publish_release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/publish_release.sh b/dev/publish_release.sh index 04cc061974e..7992be6d0f3 100755 --- a/dev/publish_release.sh +++ b/dev/publish_release.sh @@ -42,7 +42,7 @@ export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512m" RELEASE_VERSION="$1" GIT_TAG="$2" -PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" +PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr -Pgeode -Pscalding" PROJECT_OPTIONS="-pl !zeppelin-distribution" NEXUS_STAGING="https://repository.apache.org/service/local/staging" NEXUS_PROFILE="153446d1ac37c4" From c0ea07c02e2273882cb299b28031897ff3234b9a Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Sun, 19 Jun 2016 23:41:32 -0700 Subject: [PATCH 10/12] Fix wrong indentation --- zeppelin-distribution/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-distribution/pom.xml b/zeppelin-distribution/pom.xml index 9bc4ac152eb..32b05d6e994 100644 --- a/zeppelin-distribution/pom.xml +++ b/zeppelin-distribution/pom.xml @@ -73,7 +73,7 @@ - + maven-dependency-plugin From cd4cbcdefcf3a2478aad801834d3f4eaf7dffd12 Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Sun, 19 Jun 2016 23:42:15 -0700 Subject: [PATCH 11/12] curl failure check --- dev/publish_release.sh | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/dev/publish_release.sh b/dev/publish_release.sh index 7992be6d0f3..99eb5596aa8 100755 --- a/dev/publish_release.sh +++ b/dev/publish_release.sh @@ -38,6 +38,8 @@ for var in GPG_PASSPHRASE ASF_USERID ASF_PASSWORD; do done export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512m" +RED='\033[0;31m' +NC='\033[0m' # No Color RELEASE_VERSION="$1" GIT_TAG="$2" @@ -47,6 +49,24 @@ PROJECT_OPTIONS="-pl !zeppelin-distribution" NEXUS_STAGING="https://repository.apache.org/service/local/staging" NEXUS_PROFILE="153446d1ac37c4" +function cleanup() { + echo "Remove working directory and maven local repository" + rm -rf ${WORKING_DIR} + rm -rf ${tmp_repo} +} + +function curl_error() { + ret=${1} + if [[ $ret -ne 0 ]]; then + echo "curl response code is: ($ret)" + echo "See https://curl.haxx.se/libcurl/c/libcurl-errors.html to know the detailed cause of error." + echo -e "${RED}Failed to publish maven artifact to staging repository." + echo -e "IMPORTANT: You will have to re-run publish_release.sh to complete maven artifact publish.${NC}" + cleanup + exit 1 + fi +} + function publish_to_maven() { cd "${WORKING_DIR}/zeppelin" @@ -60,6 +80,8 @@ function publish_to_maven() { out="$(curl -X POST -d "${repo_request}" -u "${ASF_USERID}:${ASF_PASSWORD}" \ -H 'Content-Type:application/xml' -v \ "${NEXUS_STAGING}/profiles/${NEXUS_PROFILE}/start")" + create_ret=$? + curl_error $create_ret staged_repo_id="$(echo "${out}" | sed -e 's/.*\(orgapachezeppelin-[0-9]\{4\}\).*/\1/')" echo "Created Nexus staging repository: ${staged_repo_id}" @@ -94,6 +116,8 @@ function publish_to_maven() { dest_url="${nexus_upload}/org/apache/zeppelin/$file_short" echo " Uploading ${file_short}" curl -u "${ASF_USERID}:${ASF_PASSWORD}" --upload-file "${file_short}" "${dest_url}" + upload_ret=$? + curl_error $upload_ret done echo "Closing nexus staging repository" @@ -101,13 +125,14 @@ function publish_to_maven() { out="$(curl -X POST -d "${repo_request}" -u "${ASF_USERID}:${ASF_PASSWORD}" \ -H 'Content-Type:application/xml' -v \ "${NEXUS_STAGING}}/profiles/${NEXUS_PROFILE}/finish")" + close_ret=$? + curl_error $close_ret echo "Closed Nexus staging repository: ${staged_repo_id}" popd - rm -rf "${tmp_repo}" + echo "Complete publishing maven artifacts to apache staging repository" + echo "Once release candidate pass the vote, do not forget to hit the release button in https://repository.apache.org" } git_clone publish_to_maven - -# remove working directory -rm -rf "${WORKING_DIR}/zeppelin" +cleanup From b0e8e67669d561e5cd9683c3e293413bd2cc44ad Mon Sep 17 00:00:00 2001 From: Mina Lee Date: Mon, 20 Jun 2016 10:26:20 -0700 Subject: [PATCH 12/12] Revert "Add geode, scalding profile in maven artifact build" This reverts commit a88bc1dd2b200e06b32654c7a0654b11ea0421df. --- dev/publish_release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/publish_release.sh b/dev/publish_release.sh index 99eb5596aa8..e5a4419f5fd 100755 --- a/dev/publish_release.sh +++ b/dev/publish_release.sh @@ -44,7 +44,7 @@ NC='\033[0m' # No Color RELEASE_VERSION="$1" GIT_TAG="$2" -PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr -Pgeode -Pscalding" +PUBLISH_PROFILES="-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" PROJECT_OPTIONS="-pl !zeppelin-distribution" NEXUS_STAGING="https://repository.apache.org/service/local/staging" NEXUS_PROFILE="153446d1ac37c4"