From 562278722abf2cc5e10a293bc0eb63f005b94560 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 17 Jun 2020 11:09:43 +0200
Subject: [PATCH 01/35] Use GH-Actions artifacts to release python wheels

---
 .../python_release_automation.sh              |   2 +-
 .../main/scripts/build_release_candidate.sh   |  48 +++-
 .../download_github_actions_artifacts.py      | 231 ++++++++++++++++++
 .../main/scripts/sign_hash_python_wheels.sh   |  66 -----
 .../content/en/contribute/release-guide.md    |   8 +-
 5 files changed, 271 insertions(+), 84 deletions(-)
 create mode 100644 release/src/main/scripts/download_github_actions_artifacts.py
 delete mode 100755 release/src/main/scripts/sign_hash_python_wheels.sh

diff --git a/release/src/main/python-release/python_release_automation.sh b/release/src/main/python-release/python_release_automation.sh
index 4c527621608e..73f87a1530a6 100755
--- a/release/src/main/python-release/python_release_automation.sh
+++ b/release/src/main/python-release/python_release_automation.sh
@@ -19,7 +19,7 @@
 source release/src/main/python-release/run_release_candidate_python_quickstart.sh
 source release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
 
-for version in 2.7 3.5 3.6 3.7
+for version in 2.7 3.5 3.6 3.7 3.8
 do
   run_release_candidate_python_quickstart    "tar"   "python${version}"
   run_release_candidate_python_mobile_gaming "tar"   "python${version}"
diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index fdf292c8b358..ec12035dcb64 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -36,7 +36,8 @@ LOCAL_WEBSITE_REPO=beam_website_repo
 
 USER_REMOTE_URL=
 USER_GITHUB_ID=
-GIT_REPO_URL=git@github.com:apache/beam.git
+GIT_REPO_BASE_URL=apache/beam
+GIT_REPO_URL=git@github.com:${GIT_REPO_BASE_URL}.git
 ROOT_SVN_URL=https://dist.apache.org/repos/dist/dev/beam
 GIT_BEAM_ARCHIVE=https://github.com/apache/beam/archive
 GIT_BEAM_WEBSITE=https://github.com/apache/beam-site.git
@@ -154,7 +155,10 @@ if [[ $confirmation = "y" ]]; then
   rm -rf ~/${LOCAL_JAVA_STAGING_DIR}
 fi
 
-echo "[Current Step]: Stage python binaries"
+
+echo "[Current Step]: Stage python binaries and wheels"
+echo "===============================Pre-requirements========================"
+echo "Please make sure you have configured and started your gpg by running ./preparation_before_release.sh."
 echo "Do you want to proceed? [y|N]"
 read confirmation
 if [[ $confirmation = "y" ]]; then
@@ -170,26 +174,50 @@ if [[ $confirmation = "y" ]]; then
   git clone ${GIT_REPO_URL}
   cd ${BEAM_ROOT_DIR}
   git checkout ${RELEASE_BRANCH}
+  git push origin "${RELEASE_BRANCH}"
+  RELEASE_COMMIT=$(git rev-parse --verify HEAD)
 
-  echo '-------------------Generating Python Artifacts-----------------'
-  cd sdks/python
-  virtualenv ${LOCAL_PYTHON_VIRTUALENV}
+  echo '-------------------Creating Python Virtualenv-----------------'
+  python3 -m venv ${LOCAL_PYTHON_VIRTUALENV}
   source ${LOCAL_PYTHON_VIRTUALENV}/bin/activate
-  pip install -r build-requirements.txt
-  python setup.py sdist --format=zip
-  cd dist
+  pip install requests python-dateutil
+
+  echo '--------------Fetching GitHub Actions Artifacts--------------'
+  python release/src/main/scripts/download_github_actions_artifacts.py \
+    --github-token ${GITHUB_TOKEN} \
+    --github-user ${USER_GITHUB_ID} \
+    --repo-url ${GIT_REPO_BASE_URL} \
+    --release-branch ${RELEASE_BRANCH} \
+    --release-commit ${RELEASE_COMMIT} \
+    --artifacts_dir ${PYTHON_ARTIFACTS_DIR}
 
   svn co https://dist.apache.org/repos/dist/dev/beam
   mkdir -p beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}
-  cp apache-beam-${RELEASE}.zip beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}/apache-beam-${RELEASE}.zip
+  cp -ar ${PYTHON_ARTIFACTS_DIR}/ beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}/
   cd beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}
 
   echo "------Signing Source Release apache-beam-${RELEASE}.zip------"
   gpg --local-user ${SIGNING_KEY} --armor --detach-sig apache-beam-${RELEASE}.zip
 
-  echo "------Creating Hash Value for apache-beam-${RELEASE}.zip------"
+  echo "------Creating Hash Value for apache-beam-${RELEASE}.zip-----"
+  sha512sum apache-beam-${RELEASE}.zip > apache-beam-${RELEASE}.zip.sha512
+
+  echo "-----Signing Source Release apache-beam-${RELEASE}.tar.gz-----"
+  gpg --local-user ${SIGNING_KEY} --armor --detach-sig apache-beam-${RELEASE}.zip
+
+  echo "-----Creating Hash Value for apache-beam-${RELEASE}.tar.gz----"
   sha512sum apache-beam-${RELEASE}.zip > apache-beam-${RELEASE}.zip.sha512
 
+  echo "---------Signing Release wheels apache-beam-${RELEASE}--------"
+  for artifact in *.whl; do
+    gpg --local-user ${SIGNING_KEY} --armor --detach-sig $artifact
+  done
+
+  echo "-----Creating Hash Value for apache-beam-${RELEASE} wheels-----"
+  for artifact in *.whl; do
+    sha512sum $artifact > ${artifact}.sha512
+  done
+
   cd ..
   svn add --force ${PYTHON_ARTIFACTS_DIR}
   svn status
diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
new file mode 100644
index 000000000000..908f1f89ee86
--- /dev/null
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -0,0 +1,231 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Script for downloading GitHub Actions artifacts from 'Build python wheels' workflow."""
+import argparse
+import itertools
+import os
+import shutil
+import tempfile
+import time
+import zipfile
+
+import dateutil.parser
+import requests
+
+GH_API_URL_WORKLOW_FMT = (
+    "https://api.github.com/repos/{repo_url}/actions/workflows/build_wheels.yml"
+)
+GH_API_URL_WORKFLOW_RUNS_FMT = (
+    "https://api.github.com/repos/{repo_url}/actions/workflows/{workflow_id}/runs"
+)
+GH_WEB_URL_WORKLOW_RUN_FMT = "https://github.com/TobKed/beam/actions/runs/{workflow_id}"
+
+
+def parse_arguments():
+  parser = argparse.ArgumentParser(
+      description=
+      "Script for downloading GitHub Actions artifacts from 'Build python wheels' workflow."
+  )
+  parser.add_argument("--github-token", required=True)
+  parser.add_argument("--github-user", required=True)
+  parser.add_argument("--repo-url", required=True)
+  parser.add_argument("--release-branch", required=True)
+  parser.add_argument("--release-commit", required=True)
+  parser.add_argument("--artifacts_dir", required=True)
+
+  args = parser.parse_args()
+
+  global GITHUB_TOKEN, USER_GITHUB_ID, REPO_URL, RELEASE_BRANCH, RELEASE_COMMIT, ARTIFACTS_DIR
+  GITHUB_TOKEN = args.github_token
+  USER_GITHUB_ID = args.github_user
+  REPO_URL = args.repo_url
+  RELEASE_BRANCH = args.release_branch
+  RELEASE_COMMIT = args.release_commit
+  ARTIFACTS_DIR = args.artifacts_dir
+
+
+def requester(url, *args, return_raw_request=False, **kwargs):
+  """Helper function form making requests authorized by GitHub token"""
+  r = requests.get(url, *args, auth=("token", GITHUB_TOKEN), **kwargs)
+  r.raise_for_status()
+  if return_raw_request:
+    return r
+  return r.json()
+
+
+def yes_or_no(question):
+  """Helper function to ask yes or no question"""
+  reply = str(input(question + " (y/n): ")).lower().strip()
+  if reply == "y":
+    return True
+  if reply == "n":
+    return False
+  else:
+    return yes_or_no("Uhhhh... please enter ")
+
+
+def get_build_wheels_workflow_id():
+  url = GH_API_URL_WORKLOW_FMT.format(repo_url=REPO_URL)
+  data = requester(url)
+  return data["id"]
+
+
+def get_last_run(workflow_id):
+  url = GH_API_URL_WORKFLOW_RUNS_FMT.format(
+      repo_url=REPO_URL, workflow_id=workflow_id)
+  event_types = ["push", "pull_request"]
+  runs = []
+  for event in event_types:
+    data = requester(
+        url,
+        params={
+            "event": event, "branch": RELEASE_BRANCH
+        },
+    )
+    runs.extend(data["workflow_runs"])
+
+  filtered_commit_runs = list(
+      filter(lambda w: w.get("head_sha", "") == RELEASE_COMMIT, runs))
+  if not filtered_commit_runs:
+    workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
+        workflow_id=workflow_id)
+    raise Exception(f"No runs for workflow. Verify at {workflow_web_url}")
+
+  sorted_runs = sorted(
+      filtered_commit_runs,
+      key=lambda w: dateutil.parser.parse(w["created_at"]),
+      reverse=True,
+  )
+  last_run = sorted_runs[0]
+  print(
+      f"Found last run. SHA: {RELEASE_COMMIT}, created_at: '{last_run['created_at']}', id: {last_run['id']}"
+  )
+  workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
+      workflow_id=last_run["id"])
+  print(f"Verify at {workflow_web_url}")
+  print(
+      f"Optional upload to GCS will be available at:\n"
+      f"\tgs://beam-wheels-staging/{RELEASE_BRANCH}/{RELEASE_COMMIT}-{workflow_id}/"
+  )
+  return last_run
+
+
+def validate_run(run_data):
+  status = run_data["status"]
+  conclusion = run_data["conclusion"]
+  if status == "completed" and conclusion == "success":
+    return run_data
+
+  url = run_data["url"]
+  workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
+      workflow_id=run_data["id"])
+  print(
+      f"Waiting for Workflow run {run_data['id']} to finish. Check on {workflow_web_url}"
+  )
+  start_time = time.time()
+  last_request = start_time
+  spinner = itertools.cycle(["|", "/", "-", "\\"])
+
+  while True:
+    now = time.time()
+    elapsed_time = time.strftime("%H:%M:%S", time.gmtime(now - start_time))
+    print(
+        f"\r {next(spinner)} Waiting to finish. Elapsed time: {elapsed_time}. "
+        f"Current state: status: `{status}`, conclusion: `{conclusion}`.",
+        end="",
+    )
+
+    time.sleep(0.3)
+    if (now - last_request) > 10:
+      last_request = now
+      run_data = requester(url)
+      status = run_data["status"]
+      conclusion = run_data["conclusion"]
+      if status != "completed":
+        continue
+      elif conclusion == "success":
+        print(
+            f"\rFinished in: {elapsed_time}. "
+            f"Last state: status: `{status}`, conclusion: `{conclusion}`.",
+        )
+        return run_data
+      elif conclusion:
+        print("\r")
+        raise Exception(run_data)
+
+
+def reset_directory():
+  question = (
+      f"Artifacts directory will be cleared. Is it OK for you?\n"
+      f"Artifacts directory: {ARTIFACTS_DIR}\n"
+      f"Your answer")
+  if yes_or_no(question):
+    print(f"Clearing directory: {ARTIFACTS_DIR}")
+    shutil.rmtree(ARTIFACTS_DIR, ignore_errors=True)
+    os.makedirs(ARTIFACTS_DIR)
+  else:
+    print("You said NO for clearing artifacts directory. Quitting ...")
+    quit(1)
+
+
+def download_artifacts(artifacts_url):
+  print("Starting downloading artifacts ... (it may take a while)")
+  data_artifacts = requester(artifacts_url)
+  filtered_artifacts = [
+      a for a in data_artifacts["artifacts"] if (
+          a["name"].startswith("source_gztar_zip") or
+          a["name"].startswith("wheelhouse"))
+  ]
+  for artifact in filtered_artifacts:
+    url = artifact["archive_download_url"]
+    name = artifact["name"]
+    artifacts_size_mb = round(artifact["size_in_bytes"] / (1024 * 1024), 2)
+    print(
+        f"\tDownloading {name}.zip artifact (size: {artifacts_size_mb} megabytes)"
+    )
+    r = requester(url, return_raw_request=True, allow_redirects=True)
+
+    with tempfile.NamedTemporaryFile(
+        "wb",
+        prefix=name,
+        suffix=".zip",
+    ) as f:
+      f.write(r.content)
+
+      with zipfile.ZipFile(f.name, "r") as zip_ref:
+        print(f"\tUnzipping {len(zip_ref.filelist)} files")
+        zip_ref.extractall(ARTIFACTS_DIR)
+
+
+if __name__ == "__main__":
+  print(
+      "Starting script for download GitHub Actions artifacts for Build Wheels workflow"
+  )
+  parse_arguments()
+
+  try:
+    workflow_id = get_build_wheels_workflow_id()
+    run = get_last_run(workflow_id)
+    run = validate_run(run)
+    artifacts_url = run["artifacts_url"]
+    reset_directory()
+    download_artifacts(artifacts_url)
+    print("Script finished successfully!")
+    print(f"Artifacts available in directory: {ARTIFACTS_DIR}")
+  except KeyboardInterrupt as e:
+    print("\nScript cancelled. Quitting ...")
diff --git a/release/src/main/scripts/sign_hash_python_wheels.sh b/release/src/main/scripts/sign_hash_python_wheels.sh
deleted file mode 100755
index 05b703d2d86d..000000000000
--- a/release/src/main/scripts/sign_hash_python_wheels.sh
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/bin/bash
-#
-#    Licensed to the Apache Software Foundation (ASF) under one or more
-#    contributor license agreements.  See the NOTICE file distributed with
-#    this work for additional information regarding copyright ownership.
-#    The ASF licenses this file to You under the Apache License, Version 2.0
-#    (the "License"); you may not use this file except in compliance with
-#    the License.  You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-# This script will sign and hash python wheels.
-set -e
-
-BEAM_SVN_DIR=https://dist.apache.org/repos/dist/dev/beam
-VERSION=
-PYTHON_ARTIFACTS_DIR=python
-
-echo "===============================Pre-requirements========================"
-echo "Please make sure you have built python wheels."
-echo "Please make sure you have configured and started your gpg by running ./preparation_before_release.sh."
-echo "Do you want to proceed? [y|N]"
-read confirmation
-if [[ $confirmation != "y" ]]; then
-  echo "Please follow the release guide to build python wheels first."
-  exit
-fi
-
-echo "[Input Required] Please enter the release version:"
-read VERSION
-
-echo "================Listing all GPG keys================="
-gpg --list-keys --keyid-format LONG --fingerprint --fingerprint
-echo "Please copy the public key which is associated with your Apache account:"
-
-read SIGNING_KEY
-
-cd ~
-if [[ -d ${VERSION} ]]; then
-  rm -rf ${VERSION}
-fi
-
-svn co ${BEAM_SVN_DIR}/${VERSION}
-cd ${VERSION}/${PYTHON_ARTIFACTS_DIR}
-
-echo "Fetch wheels artifacts"
-gsutil cp -r gs://beam-wheels-staging/apache_beam-${VERSION}\*.whl .
-
-echo "Start signing and hashing python wheels artifacts"
-rm *.whl.asc || true
-rm *.whl.sha512 ||true
-for artifact in *.whl; do
-  gpg --local-user ${SIGNING_KEY} --armor --detach-sig $artifact
-  sha512sum $artifact > ${artifact}.sha512
-done
-svn add --force .
-svn commit --no-auth-cache
-
-rm -rf ~/${VERSION}
diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md
index 6e2f5512c86b..845cda0c1e5f 100644
--- a/website/www/site/content/en/contribute/release-guide.md
+++ b/website/www/site/content/en/contribute/release-guide.md
@@ -562,7 +562,7 @@ For this step, we recommend you using automation script to create a RC, but you
   1. Run gradle release to create rc tag and push source release into github repo.
   1. Run gradle publish to push java artifacts into Maven staging repo.
   1. Stage source release into dist.apache.org dev [repo](https://dist.apache.org/repos/dist/dev/beam/).
-  1. Stage,sign and hash python binaries into dist.apache.ord dev repo python dir
+  1. Stage, sign and hash python binaries and wheels into dist.apache.ord dev repo python dir
   1. Stage SDK docker images to [docker hub Apache organization](https://hub.docker.com/search?q=apache%2Fbeam&type=image).
   1. Create a PR to update beam-site, changes includes:
      * Copy python doc into beam-site
@@ -595,12 +595,6 @@ For this step, we recommend you using automation script to create a RC, but you
       1. Click the Close button.
       1. When prompted for a description, enter “Apache Beam, version X, release candidate Y”.
       1. Review all staged artifacts on https://repository.apache.org/content/repositories/orgapachebeam-NNNN/. They should contain all relevant parts for each module, including `pom.xml`, jar, test jar, javadoc, etc. Artifact names should follow [the existing format](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22) in which artifact name mirrors directory structure, e.g., `beam-sdks-java-io-kafka`. Carefully review any new artifacts.
-  1. Build and stage python wheels.
-      - There is a wrapper repo [beam-wheels](https://github.com/apache/beam-wheels) to help build python wheels.
-      - If you are interested in how it works, please refer to the [structure section](https://github.com/apache/beam-wheels#structure).
-      - Please follow the [user guide](https://github.com/apache/beam-wheels#user-guide) to build python wheels.
-      - Once all python wheels have been staged to GCS,
-      please run [./sign_hash_python_wheels.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/sign_hash_python_wheels.sh), which copies the wheels along with signatures and hashes to [dist.apache.org](https://dist.apache.org/repos/dist/dev/beam/).
 
 **********
 

From 162e5814c95fa1174ee4aafaed7b318b3224b089 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Thu, 2 Jul 2020 13:43:56 +0200
Subject: [PATCH 02/35] fixup! Use GH-Actions artifacts to release python
 wheels

---
 .../scripts/download_github_actions_artifacts.py    | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 908f1f89ee86..3492175bb429 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -33,7 +33,7 @@
 GH_API_URL_WORKFLOW_RUNS_FMT = (
     "https://api.github.com/repos/{repo_url}/actions/workflows/{workflow_id}/runs"
 )
-GH_WEB_URL_WORKLOW_RUN_FMT = "https://github.com/TobKed/beam/actions/runs/{workflow_id}"
+GH_WEB_URL_WORKLOW_RUN_FMT = "https://github.com/apache/beam/actions/runs/{workflow_id}"
 
 
 def parse_arguments():
@@ -59,7 +59,7 @@ def parse_arguments():
   ARTIFACTS_DIR = args.artifacts_dir
 
 
-def requester(url, *args, return_raw_request=False, **kwargs):
+def requester(url, return_raw_request=False, *args, **kwargs):
   """Helper function form making requests authorized by GitHub token"""
   r = requests.get(url, *args, auth=("token", GITHUB_TOKEN), **kwargs)
   r.raise_for_status()
@@ -104,7 +104,9 @@ def get_last_run(workflow_id):
   if not filtered_commit_runs:
     workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
         workflow_id=workflow_id)
-    raise Exception(f"No runs for workflow. Verify at {workflow_web_url}")
+    raise Exception(
+        f"No runs for workflow (branch {RELEASE_BRANCH}, commit {RELEASE_COMMIT}). Verify at {workflow_web_url}"
+    )
 
   sorted_runs = sorted(
       filtered_commit_runs,
@@ -135,7 +137,7 @@ def validate_run(run_data):
   workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
       workflow_id=run_data["id"])
   print(
-      f"Waiting for Workflow run {run_data['id']} to finish. Check on {workflow_web_url}"
+      f"Started waiting for Workflow run {run_data['id']} to finish. Check on {workflow_web_url}"
   )
   start_time = time.time()
   last_request = start_time
@@ -166,7 +168,8 @@ def validate_run(run_data):
         return run_data
       elif conclusion:
         print("\r")
-        raise Exception(run_data)
+        raise Exception(
+            f"Run unsuccessful. Conclusion: {conclusion}. Payload: {run_data}")
 
 
 def reset_directory():

From 7a7f18d9e5ee91e3a1fb4fdeb746b757cb0201d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Thu, 2 Jul 2020 14:18:42 +0200
Subject: [PATCH 03/35] fixup! fixup! Use GH-Actions artifacts to release
 python wheels

---
 .../src/main/scripts/download_github_actions_artifacts.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 3492175bb429..48561da80c47 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -33,7 +33,7 @@
 GH_API_URL_WORKFLOW_RUNS_FMT = (
     "https://api.github.com/repos/{repo_url}/actions/workflows/{workflow_id}/runs"
 )
-GH_WEB_URL_WORKLOW_RUN_FMT = "https://github.com/apache/beam/actions/runs/{workflow_id}"
+GH_WEB_URL_WORKLOW_RUN_FMT = "https://github.com/{repo_url}/actions/runs/{workflow_id}"
 
 
 def parse_arguments():
@@ -103,7 +103,7 @@ def get_last_run(workflow_id):
       filter(lambda w: w.get("head_sha", "") == RELEASE_COMMIT, runs))
   if not filtered_commit_runs:
     workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
-        workflow_id=workflow_id)
+        repo_url=REPO_URL, workflow_id=workflow_id)
     raise Exception(
         f"No runs for workflow (branch {RELEASE_BRANCH}, commit {RELEASE_COMMIT}). Verify at {workflow_web_url}"
     )
@@ -118,7 +118,7 @@ def get_last_run(workflow_id):
       f"Found last run. SHA: {RELEASE_COMMIT}, created_at: '{last_run['created_at']}', id: {last_run['id']}"
   )
   workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
-      workflow_id=last_run["id"])
+      repo_url=REPO_URL, workflow_id=last_run["id"])
   print(f"Verify at {workflow_web_url}")
   print(
       f"Optional upload to GCS will be available at:\n"
@@ -135,7 +135,7 @@ def validate_run(run_data):
 
   url = run_data["url"]
   workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
-      workflow_id=run_data["id"])
+      repo_url=REPO_URL, workflow_id=run_data["id"])
   print(
       f"Started waiting for Workflow run {run_data['id']} to finish. Check on {workflow_web_url}"
   )

From ae071a5b6cae0f0fc5db9ee66395af3d7d67cf79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Thu, 2 Jul 2020 15:47:49 +0200
Subject: [PATCH 04/35] fixup! fixup! fixup! Use GH-Actions artifacts to
 release python wheels

---
 .../download_github_actions_artifacts.py      | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 48561da80c47..b3191244a22e 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -59,7 +59,7 @@ def parse_arguments():
   ARTIFACTS_DIR = args.artifacts_dir
 
 
-def requester(url, return_raw_request=False, *args, **kwargs):
+def request_url(url, return_raw_request=False, *args, **kwargs):
   """Helper function form making requests authorized by GitHub token"""
   r = requests.get(url, *args, auth=("token", GITHUB_TOKEN), **kwargs)
   r.raise_for_status()
@@ -68,7 +68,7 @@ def requester(url, return_raw_request=False, *args, **kwargs):
   return r.json()
 
 
-def yes_or_no(question):
+def get_yes_or_no_answer(question):
   """Helper function to ask yes or no question"""
   reply = str(input(question + " (y/n): ")).lower().strip()
   if reply == "y":
@@ -76,12 +76,12 @@ def yes_or_no(question):
   if reply == "n":
     return False
   else:
-    return yes_or_no("Uhhhh... please enter ")
+    return get_yes_or_no_answer("Uhhhh... please enter ")
 
 
 def get_build_wheels_workflow_id():
   url = GH_API_URL_WORKLOW_FMT.format(repo_url=REPO_URL)
-  data = requester(url)
+  data = request_url(url)
   return data["id"]
 
 
@@ -91,7 +91,7 @@ def get_last_run(workflow_id):
   event_types = ["push", "pull_request"]
   runs = []
   for event in event_types:
-    data = requester(
+    data = request_url(
         url,
         params={
             "event": event, "branch": RELEASE_BRANCH
@@ -155,7 +155,7 @@ def validate_run(run_data):
     time.sleep(0.3)
     if (now - last_request) > 10:
       last_request = now
-      run_data = requester(url)
+      run_data = request_url(url)
       status = run_data["status"]
       conclusion = run_data["conclusion"]
       if status != "completed":
@@ -166,7 +166,7 @@ def validate_run(run_data):
             f"Last state: status: `{status}`, conclusion: `{conclusion}`.",
         )
         return run_data
-      elif conclusion:
+      else:
         print("\r")
         raise Exception(
             f"Run unsuccessful. Conclusion: {conclusion}. Payload: {run_data}")
@@ -177,7 +177,7 @@ def reset_directory():
       f"Artifacts directory will be cleared. Is it OK for you?\n"
       f"Artifacts directory: {ARTIFACTS_DIR}\n"
       f"Your answer")
-  if yes_or_no(question):
+  if get_yes_or_no_answer(question):
     print(f"Clearing directory: {ARTIFACTS_DIR}")
     shutil.rmtree(ARTIFACTS_DIR, ignore_errors=True)
     os.makedirs(ARTIFACTS_DIR)
@@ -188,7 +188,7 @@ def reset_directory():
 
 def download_artifacts(artifacts_url):
   print("Starting downloading artifacts ... (it may take a while)")
-  data_artifacts = requester(artifacts_url)
+  data_artifacts = request_url(artifacts_url)
   filtered_artifacts = [
       a for a in data_artifacts["artifacts"] if (
           a["name"].startswith("source_gztar_zip") or
@@ -201,7 +201,7 @@ def download_artifacts(artifacts_url):
     print(
         f"\tDownloading {name}.zip artifact (size: {artifacts_size_mb} megabytes)"
     )
-    r = requester(url, return_raw_request=True, allow_redirects=True)
+    r = request_url(url, return_raw_request=True, allow_redirects=True)
 
     with tempfile.NamedTemporaryFile(
         "wb",

From 49dfb8cbf8dc3bc4adab6798e7a93379231e16b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 7 Jul 2020 13:42:29 +0200
Subject: [PATCH 05/35] fixup! fixup! fixup! fixup! Use GH-Actions artifacts to
 release python wheels

---
 release/src/main/scripts/download_github_actions_artifacts.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index b3191244a22e..6f6fcfab8b6e 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -142,6 +142,7 @@ def validate_run(run_data):
   start_time = time.time()
   last_request = start_time
   spinner = itertools.cycle(["|", "/", "-", "\\"])
+  request_interval = 10
 
   while True:
     now = time.time()
@@ -153,7 +154,7 @@ def validate_run(run_data):
     )
 
     time.sleep(0.3)
-    if (now - last_request) > 10:
+    if (now - last_request) > request_interval:
       last_request = now
       run_data = request_url(url)
       status = run_data["status"]

From f2b5f0ef8052100e890ad3d74508a2a43eda561c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 7 Jul 2020 13:53:58 +0200
Subject: [PATCH 06/35] fixup! fixup! fixup! fixup! fixup! Use GH-Actions
 artifacts to release python wheels

---
 .../src/main/scripts/download_github_actions_artifacts.py   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 6f6fcfab8b6e..dc529ddd5008 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -20,6 +20,7 @@
 import itertools
 import os
 import shutil
+import sys
 import tempfile
 import time
 import zipfile
@@ -175,8 +176,7 @@ def validate_run(run_data):
 
 def reset_directory():
   question = (
-      f"Artifacts directory will be cleared. Is it OK for you?\n"
-      f"Artifacts directory: {ARTIFACTS_DIR}\n"
+      "Creating Artifacts directory. Any existing content in {ARTIFACTS_DIR} will be erased. Proceed?\n"
       f"Your answer")
   if get_yes_or_no_answer(question):
     print(f"Clearing directory: {ARTIFACTS_DIR}")
@@ -184,7 +184,7 @@ def reset_directory():
     os.makedirs(ARTIFACTS_DIR)
   else:
     print("You said NO for clearing artifacts directory. Quitting ...")
-    quit(1)
+    sys.exit(1)
 
 
 def download_artifacts(artifacts_url):

From bdb08b0aa7d07e62f28fe32eb02a4eb63bc10091 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 8 Jul 2020 16:09:23 +0200
Subject: [PATCH 07/35] Update FMT strings

---
 .../src/main/scripts/download_github_actions_artifacts.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index dc529ddd5008..37cf1bfc25b5 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -34,7 +34,7 @@
 GH_API_URL_WORKFLOW_RUNS_FMT = (
     "https://api.github.com/repos/{repo_url}/actions/workflows/{workflow_id}/runs"
 )
-GH_WEB_URL_WORKLOW_RUN_FMT = "https://github.com/{repo_url}/actions/runs/{workflow_id}"
+GH_WEB_URL_WORKLOW_RUN_FMT = "https://github.com/{repo_url}/actions/runs/{run_id}"
 
 
 def parse_arguments():
@@ -103,7 +103,7 @@ def get_last_run(workflow_id):
   filtered_commit_runs = list(
       filter(lambda w: w.get("head_sha", "") == RELEASE_COMMIT, runs))
   if not filtered_commit_runs:
-    workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
+    workflow_web_url = GH_API_URL_WORKFLOW_RUNS_FMT.format(
         repo_url=REPO_URL, workflow_id=workflow_id)
     raise Exception(
         f"No runs for workflow (branch {RELEASE_BRANCH}, commit {RELEASE_COMMIT}). Verify at {workflow_web_url}"
@@ -119,7 +119,7 @@ def get_last_run(workflow_id):
       f"Found last run. SHA: {RELEASE_COMMIT}, created_at: '{last_run['created_at']}', id: {last_run['id']}"
   )
   workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
-      repo_url=REPO_URL, workflow_id=last_run["id"])
+      repo_url=REPO_URL, run_id=last_run["id"])
   print(f"Verify at {workflow_web_url}")
   print(
       f"Optional upload to GCS will be available at:\n"
@@ -136,7 +136,7 @@ def validate_run(run_data):
 
   url = run_data["url"]
   workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
-      repo_url=REPO_URL, workflow_id=run_data["id"])
+      repo_url=REPO_URL, run_id=run_data["id"])
   print(
       f"Started waiting for Workflow run {run_data['id']} to finish. Check on {workflow_web_url}"
   )

From ad77697f37b5c87e2ce92be6821400be84744cc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 8 Jul 2020 16:17:20 +0200
Subject: [PATCH 08/35] fixup! Update FMT strings

---
 .../scripts/download_github_actions_artifacts.py    | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 37cf1bfc25b5..9039e6684ccc 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -60,24 +60,24 @@ def parse_arguments():
   ARTIFACTS_DIR = args.artifacts_dir
 
 
-def request_url(url, return_raw_request=False, *args, **kwargs):
+def request_url(url, return_raw_response=False, *args, **kwargs):
   """Helper function form making requests authorized by GitHub token"""
   r = requests.get(url, *args, auth=("token", GITHUB_TOKEN), **kwargs)
   r.raise_for_status()
-  if return_raw_request:
+  if return_raw_response:
     return r
   return r.json()
 
 
 def get_yes_or_no_answer(question):
   """Helper function to ask yes or no question"""
-  reply = str(input(question + " (y/n): ")).lower().strip()
+  reply = str(input(question + " 'y' or 'n'): ")).lower().strip()
   if reply == "y":
     return True
   if reply == "n":
     return False
   else:
-    return get_yes_or_no_answer("Uhhhh... please enter ")
+    return get_yes_or_no_answer("Uhhhh... please enter")
 
 
 def get_build_wheels_workflow_id():
@@ -100,8 +100,7 @@ def get_last_run(workflow_id):
     )
     runs.extend(data["workflow_runs"])
 
-  filtered_commit_runs = list(
-      filter(lambda w: w.get("head_sha", "") == RELEASE_COMMIT, runs))
+  filtered_commit_runs = [r for r in runs if r.get("head_sha", "") == RELEASE_COMMIT]
   if not filtered_commit_runs:
     workflow_web_url = GH_API_URL_WORKFLOW_RUNS_FMT.format(
         repo_url=REPO_URL, workflow_id=workflow_id)
@@ -202,7 +201,7 @@ def download_artifacts(artifacts_url):
     print(
         f"\tDownloading {name}.zip artifact (size: {artifacts_size_mb} megabytes)"
     )
-    r = request_url(url, return_raw_request=True, allow_redirects=True)
+    r = request_url(url, return_raw_response=True, allow_redirects=True)
 
     with tempfile.NamedTemporaryFile(
         "wb",

From e19df43efe54d59eaf5a597231e4931ddf919bd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Mon, 13 Jul 2020 16:36:38 +0200
Subject: [PATCH 09/35] Downloading files in streaming mode

---
 .../scripts/download_github_actions_artifacts.py    | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 9039e6684ccc..362e90400df1 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -201,15 +201,12 @@ def download_artifacts(artifacts_url):
     print(
         f"\tDownloading {name}.zip artifact (size: {artifacts_size_mb} megabytes)"
     )
-    r = request_url(url, return_raw_response=True, allow_redirects=True)
-
-    with tempfile.NamedTemporaryFile(
-        "wb",
-        prefix=name,
-        suffix=".zip",
-    ) as f:
-      f.write(r.content)
 
+    with tempfile.NamedTemporaryFile("wb", prefix=name, suffix=".zip") as f, requests.get(
+       url, auth=("token", GITHUB_TOKEN), allow_redirects=True, stream=True
+    ) as r:
+      with open(f.name, 'wb') as f:
+        shutil.copyfileobj(r.raw, f)
       with zipfile.ZipFile(f.name, "r") as zip_ref:
         print(f"\tUnzipping {len(zip_ref.filelist)} files")
         zip_ref.extractall(ARTIFACTS_DIR)

From 1c27e5ef4264e4e47bd1ebfa45f974f0cc816011 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Mon, 13 Jul 2020 16:46:51 +0200
Subject: [PATCH 10/35] fixup! Downloading files in streaming mode

---
 .../scripts/download_github_actions_artifacts.py   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 362e90400df1..b56d8ef936af 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -60,13 +60,13 @@ def parse_arguments():
   ARTIFACTS_DIR = args.artifacts_dir
 
 
-def request_url(url, return_raw_response=False, *args, **kwargs):
+def request_url(url, return_json=True, *args, **kwargs):
   """Helper function form making requests authorized by GitHub token"""
   r = requests.get(url, *args, auth=("token", GITHUB_TOKEN), **kwargs)
-  r.raise_for_status()
-  if return_raw_response:
-    return r
-  return r.json()
+  if return_json:
+    r.raise_for_status()
+    return r.json()
+  return r
 
 
 def get_yes_or_no_answer(question):
@@ -202,8 +202,8 @@ def download_artifacts(artifacts_url):
         f"\tDownloading {name}.zip artifact (size: {artifacts_size_mb} megabytes)"
     )
 
-    with tempfile.NamedTemporaryFile("wb", prefix=name, suffix=".zip") as f, requests.get(
-       url, auth=("token", GITHUB_TOKEN), allow_redirects=True, stream=True
+    with tempfile.NamedTemporaryFile("wb", prefix=name, suffix=".zip") as f, request_url(
+       url, return_json=False, allow_redirects=True, stream=True
     ) as r:
       with open(f.name, 'wb') as f:
         shutil.copyfileobj(r.raw, f)

From 0348b1f28591e5a210328111bacbb28f017ca279 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Mon, 13 Jul 2020 17:26:02 +0200
Subject: [PATCH 11/35] Support only push trigger events

---
 .../download_github_actions_artifacts.py      | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index b56d8ef936af..ce3114d48de4 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -89,18 +89,18 @@ def get_build_wheels_workflow_id():
 def get_last_run(workflow_id):
   url = GH_API_URL_WORKFLOW_RUNS_FMT.format(
       repo_url=REPO_URL, workflow_id=workflow_id)
-  event_types = ["push", "pull_request"]
-  runs = []
-  for event in event_types:
-    data = request_url(
-        url,
-        params={
-            "event": event, "branch": RELEASE_BRANCH
-        },
-    )
-    runs.extend(data["workflow_runs"])
+  event_type = "push"
+  data = request_url(
+      url,
+      params={
+          "event": event_type, "branch": RELEASE_BRANCH
+      },
+  )
+  runs = data["workflow_runs"]
 
-  filtered_commit_runs = [r for r in runs if r.get("head_sha", "") == RELEASE_COMMIT]
+  filtered_commit_runs = [
+      r for r in runs if r.get("head_sha", "") == RELEASE_COMMIT
+  ]
   if not filtered_commit_runs:
     workflow_web_url = GH_API_URL_WORKFLOW_RUNS_FMT.format(
         repo_url=REPO_URL, workflow_id=workflow_id)

From 025e247314c3150cc222a9daf154a67c736afb2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Mon, 13 Jul 2020 17:28:13 +0200
Subject: [PATCH 12/35] Change GCS upload message

---
 release/src/main/scripts/download_github_actions_artifacts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index ce3114d48de4..e0350845402f 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -121,7 +121,7 @@ def get_last_run(workflow_id):
       repo_url=REPO_URL, run_id=last_run["id"])
   print(f"Verify at {workflow_web_url}")
   print(
-      f"Optional upload to GCS will be available at:\n"
+      f"Upload to GCS available at:\n"
       f"\tgs://beam-wheels-staging/{RELEASE_BRANCH}/{RELEASE_COMMIT}-{workflow_id}/"
   )
   return last_run

From 9f687e52465ed30b299ecda60a99d9c12e417e78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Mon, 13 Jul 2020 18:49:07 +0200
Subject: [PATCH 13/35] More explicit check of workflow run status and
 conclusion

---
 release/src/main/scripts/download_github_actions_artifacts.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index e0350845402f..6f43e932bdf4 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -159,9 +159,9 @@ def validate_run(run_data):
       run_data = request_url(url)
       status = run_data["status"]
       conclusion = run_data["conclusion"]
-      if status != "completed":
+      if status in ["queued", "in_progress"]:
         continue
-      elif conclusion == "success":
+      elif status == "completed" and conclusion == "success":
         print(
             f"\rFinished in: {elapsed_time}. "
             f"Last state: status: `{status}`, conclusion: `{conclusion}`.",

From dca7f84112f2c4c64676fc131218c9266b2bb4b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 14 Jul 2020 15:01:37 +0200
Subject: [PATCH 14/35] Refactor to do not use global vars, ask for gh-token

---
 .../main/scripts/build_release_candidate.sh   |   1 -
 .../download_github_actions_artifacts.py      | 167 ++++++++++++------
 2 files changed, 112 insertions(+), 56 deletions(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index ec12035dcb64..05be0434c08f 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -184,7 +184,6 @@ if [[ $confirmation = "y" ]]; then
 
   echo '--------------Fetching GitHub Actions Artifacts--------------'
   python release/src/main/scripts/download_github_actions_artifacts.py \
-    --github-token ${GITHUB_TOKEN} \
     --github-user ${USER_GITHUB_ID} \
     --repo-url ${GIT_REPO_BASE_URL} \
     --release-branch ${RELEASE_BRANCH} \
diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 6f43e932bdf4..ca97bb85fb4c 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -19,6 +19,7 @@
 import argparse
 import itertools
 import os
+import pprint
 import shutil
 import sys
 import tempfile
@@ -38,11 +39,14 @@
 
 
 def parse_arguments():
+  """
+  Gets all neccessary data from the user by parsing arguments or asking for input.
+  Return: github_token, user_github_id, repo_url, release_branch, release_commit, artifacts_dir
+  """
   parser = argparse.ArgumentParser(
       description=
       "Script for downloading GitHub Actions artifacts from 'Build python wheels' workflow."
   )
-  parser.add_argument("--github-token", required=True)
   parser.add_argument("--github-user", required=True)
   parser.add_argument("--repo-url", required=True)
   parser.add_argument("--release-branch", required=True)
@@ -51,26 +55,56 @@ def parse_arguments():
 
   args = parser.parse_args()
 
-  global GITHUB_TOKEN, USER_GITHUB_ID, REPO_URL, RELEASE_BRANCH, RELEASE_COMMIT, ARTIFACTS_DIR
-  GITHUB_TOKEN = args.github_token
-  USER_GITHUB_ID = args.github_user
-  REPO_URL = args.repo_url
-  RELEASE_BRANCH = args.release_branch
-  RELEASE_COMMIT = args.release_commit
-  ARTIFACTS_DIR = args.artifacts_dir
+  github_token = ask_for_github_token()
+  print("You passed following arguments:")
+  pprint.pprint({**vars(args), **{"github_token": github_token}})
+  if not get_yes_or_no_answer("Do you want to continue?"):
+    print("You said NO. Quitting ...")
+    sys.exit(1)
+
+  user_github_id = args.github_user
+  repo_url = args.repo_url
+  release_branch = args.release_branch
+  release_commit = args.release_commit
+  artifacts_dir = args.artifacts_dir
+
+  return github_token, user_github_id, repo_url, release_branch, release_commit, artifacts_dir
+
 
+def ask_for_github_token():
+  """Ask for github token and print basic information about it"""
+  url = "https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token"
+  message = (
+      f"You need to have a github access token with public_repo scope. "
+      f"More info about creating access tokens can be found here {url}")
+  print(message)
+  github_token = input("Enter github token: ")
+  if not github_token:
+    return ask_for_github_token()
+  return github_token
 
-def request_url(url, return_json=True, *args, **kwargs):
+
+def request_url(url, github_token, return_json=True, *args, **kwargs):
   """Helper function form making requests authorized by GitHub token"""
-  r = requests.get(url, *args, auth=("token", GITHUB_TOKEN), **kwargs)
+  r = requests.get(url, *args, auth=("token", github_token), **kwargs)
   if return_json:
     r.raise_for_status()
     return r.json()
   return r
 
 
+def safe_get(data, key, url=None):
+  """Gets data by the key with informative Exception in case of non existent key."""
+  if key not in data:
+    message = f'There is missing key: "{key}" in response data: {data}.'
+    if url:
+      message += f" Requested url: {url}"
+    raise ValueError(message)
+  return data.get(key)
+
+
 def get_yes_or_no_answer(question):
-  """Helper function to ask yes or no question"""
+  """Asks yes or no question"""
   reply = str(input(question + " 'y' or 'n'): ")).lower().strip()
   if reply == "y":
     return True
@@ -80,32 +114,41 @@ def get_yes_or_no_answer(question):
     return get_yes_or_no_answer("Uhhhh... please enter")
 
 
-def get_build_wheels_workflow_id():
-  url = GH_API_URL_WORKLOW_FMT.format(repo_url=REPO_URL)
-  data = request_url(url)
-  return data["id"]
+def get_build_wheels_workflow_id(repo_url, github_token):
+  """Gets workflow id"""
+  url = GH_API_URL_WORKLOW_FMT.format(repo_url=repo_url)
+  data = request_url(url, github_token)
+  return safe_get(data, "id", url)
 
 
-def get_last_run(workflow_id):
+def get_last_run(
+    workflow_id, repo_url, release_branch, release_commit, github_token):
+  """
+  Gets data of last run for given repo, branch and commit.
+  Raises exception when no run found.
+  """
   url = GH_API_URL_WORKFLOW_RUNS_FMT.format(
-      repo_url=REPO_URL, workflow_id=workflow_id)
-  event_type = "push"
+      repo_url=repo_url, workflow_id=workflow_id)
   data = request_url(
       url,
+      github_token,
       params={
-          "event": event_type, "branch": RELEASE_BRANCH
+          # FIXME change to push
+          "event": "pull_request",
+          "branch": release_branch
+          # "event": "push", "branch": release_branch
       },
   )
-  runs = data["workflow_runs"]
+  runs = safe_get(data, "workflow_runs", url)
 
   filtered_commit_runs = [
-      r for r in runs if r.get("head_sha", "") == RELEASE_COMMIT
+      r for r in runs if r.get("head_sha", "") == release_commit
   ]
   if not filtered_commit_runs:
     workflow_web_url = GH_API_URL_WORKFLOW_RUNS_FMT.format(
-        repo_url=REPO_URL, workflow_id=workflow_id)
+        repo_url=repo_url, workflow_id=workflow_id)
     raise Exception(
-        f"No runs for workflow (branch {RELEASE_BRANCH}, commit {RELEASE_COMMIT}). Verify at {workflow_web_url}"
+        f"No runs for workflow (branch {release_branch}, commit {release_commit}). Verify at {workflow_web_url}"
     )
 
   sorted_runs = sorted(
@@ -115,27 +158,28 @@ def get_last_run(workflow_id):
   )
   last_run = sorted_runs[0]
   print(
-      f"Found last run. SHA: {RELEASE_COMMIT}, created_at: '{last_run['created_at']}', id: {last_run['id']}"
+      f"Found last run. SHA: {release_commit}, created_at: '{last_run['created_at']}', id: {last_run['id']}"
   )
   workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
-      repo_url=REPO_URL, run_id=last_run["id"])
+      repo_url=repo_url, run_id=last_run["id"])
   print(f"Verify at {workflow_web_url}")
   print(
       f"Upload to GCS available at:\n"
-      f"\tgs://beam-wheels-staging/{RELEASE_BRANCH}/{RELEASE_COMMIT}-{workflow_id}/"
+      f"\tgs://beam-wheels-staging/{release_branch}/{release_commit}-{workflow_id}/"
   )
   return last_run
 
 
-def validate_run(run_data):
-  status = run_data["status"]
-  conclusion = run_data["conclusion"]
+def validate_run(run_data, repo_url, github_token):
+  """Validates workflow run. Verifies status and waits if run is not finished."""
+  status = safe_get(run_data, "status")
+  conclusion = safe_get(run_data, "conclusion")
   if status == "completed" and conclusion == "success":
     return run_data
 
-  url = run_data["url"]
+  url = safe_get(run_data, "url")
   workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
-      repo_url=REPO_URL, run_id=run_data["id"])
+      repo_url=repo_url, run_id=run_data["id"])
   print(
       f"Started waiting for Workflow run {run_data['id']} to finish. Check on {workflow_web_url}"
   )
@@ -156,9 +200,9 @@ def validate_run(run_data):
     time.sleep(0.3)
     if (now - last_request) > request_interval:
       last_request = now
-      run_data = request_url(url)
-      status = run_data["status"]
-      conclusion = run_data["conclusion"]
+      run_data = request_url(url, github_token)
+      status = safe_get(run_data, "status")
+      conclusion = safe_get(run_data, "conclusion")
       if status in ["queued", "in_progress"]:
         continue
       elif status == "completed" and conclusion == "success":
@@ -173,59 +217,72 @@ def validate_run(run_data):
             f"Run unsuccessful. Conclusion: {conclusion}. Payload: {run_data}")
 
 
-def reset_directory():
+def reset_directory(artifacts_dir):
+  """Clears directory asking for confirmation before."""
   question = (
-      "Creating Artifacts directory. Any existing content in {ARTIFACTS_DIR} will be erased. Proceed?\n"
+      f"Creating Artifacts directory. Any existing content in {artifacts_dir} will be erased. Proceed?\n"
       f"Your answer")
   if get_yes_or_no_answer(question):
-    print(f"Clearing directory: {ARTIFACTS_DIR}")
-    shutil.rmtree(ARTIFACTS_DIR, ignore_errors=True)
-    os.makedirs(ARTIFACTS_DIR)
+    print(f"Clearing directory: {artifacts_dir}")
+    shutil.rmtree(artifacts_dir, ignore_errors=True)
+    os.makedirs(artifacts_dir)
   else:
     print("You said NO for clearing artifacts directory. Quitting ...")
     sys.exit(1)
 
 
-def download_artifacts(artifacts_url):
+def download_artifacts(artifacts_url, artifacts_dir, github_token):
+  """Downloads github artifacts from given url."""
   print("Starting downloading artifacts ... (it may take a while)")
-  data_artifacts = request_url(artifacts_url)
+  data_artifacts = request_url(artifacts_url, github_token)
+  artifacts = safe_get(data_artifacts, "artifacts", artifacts_url)
   filtered_artifacts = [
-      a for a in data_artifacts["artifacts"] if (
+      a for a in artifacts if (
           a["name"].startswith("source_gztar_zip") or
           a["name"].startswith("wheelhouse"))
   ]
   for artifact in filtered_artifacts:
-    url = artifact["archive_download_url"]
-    name = artifact["name"]
+    url = safe_get(artifact, "archive_download_url")
+    name = safe_get(artifact, "name")
     artifacts_size_mb = round(artifact["size_in_bytes"] / (1024 * 1024), 2)
     print(
         f"\tDownloading {name}.zip artifact (size: {artifacts_size_mb} megabytes)"
     )
 
-    with tempfile.NamedTemporaryFile("wb", prefix=name, suffix=".zip") as f, request_url(
-       url, return_json=False, allow_redirects=True, stream=True
+    with tempfile.NamedTemporaryFile(
+        "wb", prefix=name, suffix=".zip"
+    ) as f, request_url(
+        url, github_token, return_json=False, allow_redirects=True, stream=True
     ) as r:
-      with open(f.name, 'wb') as f:
+      with open(f.name, "wb") as f:
         shutil.copyfileobj(r.raw, f)
       with zipfile.ZipFile(f.name, "r") as zip_ref:
         print(f"\tUnzipping {len(zip_ref.filelist)} files")
-        zip_ref.extractall(ARTIFACTS_DIR)
+        zip_ref.extractall(artifacts_dir)
 
 
 if __name__ == "__main__":
   print(
       "Starting script for download GitHub Actions artifacts for Build Wheels workflow"
   )
-  parse_arguments()
+  (
+      github_token,
+      user_github_id,
+      repo_url,
+      release_branch,
+      release_commit,
+      artifacts_dir,
+  ) = parse_arguments()
 
   try:
-    workflow_id = get_build_wheels_workflow_id()
-    run = get_last_run(workflow_id)
-    run = validate_run(run)
+    workflow_id = get_build_wheels_workflow_id(repo_url, github_token)
+    run = get_last_run(
+        workflow_id, repo_url, release_branch, release_commit, github_token)
+    run = validate_run(run, repo_url, github_token)
     artifacts_url = run["artifacts_url"]
-    reset_directory()
-    download_artifacts(artifacts_url)
+    reset_directory(artifacts_dir)
+    download_artifacts(artifacts_url, artifacts_dir, github_token)
     print("Script finished successfully!")
-    print(f"Artifacts available in directory: {ARTIFACTS_DIR}")
+    print(f"Artifacts available in directory: {artifacts_dir}")
   except KeyboardInterrupt as e:
     print("\nScript cancelled. Quitting ...")

From a1d3e00a5d9a0763798b90438c370a009f3902e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 15 Jul 2020 10:33:07 +0200
Subject: [PATCH 15/35] fixup! Refactor to do not use global vars, ask for
 gh-token

---
 .../download_github_actions_artifacts.py      | 136 +++++++++++-------
 1 file changed, 81 insertions(+), 55 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index ca97bb85fb4c..db32d619b30d 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -29,12 +29,9 @@
 import dateutil.parser
 import requests
 
-GH_API_URL_WORKLOW_FMT = (
-    "https://api.github.com/repos/{repo_url}/actions/workflows/build_wheels.yml"
-)
-GH_API_URL_WORKFLOW_RUNS_FMT = (
-    "https://api.github.com/repos/{repo_url}/actions/workflows/{workflow_id}/runs"
-)
+GH_API_URL_WORKLOW_FMT = "https://api.github.com/repos/{repo_url}/actions/workflows/build_wheels.yml"
+GH_API_URL_WORKFLOW_RUNS_FMT = "https://api.github.com/repos/{repo_url}/actions/workflows/{workflow_id}/runs"
+GH_API_URL_WORKFLOW_RUN_FMT = "https://api.github.com/repos/{repo_url}/actions/runs/{run_id}"
 GH_WEB_URL_WORKLOW_RUN_FMT = "https://github.com/{repo_url}/actions/runs/{run_id}"
 
 
@@ -54,10 +51,11 @@ def parse_arguments():
   parser.add_argument("--artifacts_dir", required=True)
 
   args = parser.parse_args()
-
   github_token = ask_for_github_token()
+
   print("You passed following arguments:")
   pprint.pprint({**vars(args), **{"github_token": github_token}})
+
   if not get_yes_or_no_answer("Do you want to continue?"):
     print("You said NO. Quitting ...")
     sys.exit(1)
@@ -72,7 +70,7 @@ def parse_arguments():
 
 
 def ask_for_github_token():
-  """Ask for github token and print basic information about it"""
+  """Ask for github token and print basic information about it."""
   url = "https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token"
   message = (
       f"You need to have a github access token with public_repo scope. "
@@ -85,7 +83,7 @@ def ask_for_github_token():
 
 
 def request_url(url, github_token, return_json=True, *args, **kwargs):
-  """Helper function form making requests authorized by GitHub token"""
+  """Helper function form making requests authorized by GitHub token."""
   r = requests.get(url, *args, auth=("token", github_token), **kwargs)
   if return_json:
     r.raise_for_status()
@@ -104,27 +102,33 @@ def safe_get(data, key, url=None):
 
 
 def get_yes_or_no_answer(question):
-  """Asks yes or no question"""
+  """Asks yes or no question."""
   reply = str(input(question + " 'y' or 'n'): ")).lower().strip()
   if reply == "y":
     return True
-  if reply == "n":
+  elif reply == "n":
     return False
   else:
     return get_yes_or_no_answer("Uhhhh... please enter")
 
 
 def get_build_wheels_workflow_id(repo_url, github_token):
-  """Gets workflow id"""
+  """Gets workflow id."""
   url = GH_API_URL_WORKLOW_FMT.format(repo_url=repo_url)
   data = request_url(url, github_token)
   return safe_get(data, "id", url)
 
 
-def get_last_run(
+def get_single_workflow_run_data(run_id, repo_url, github_token):
+  """Gets single workflow run data (github api payload)."""
+  url = GH_API_URL_WORKFLOW_RUN_FMT.format(repo_url=repo_url, run_id=run_id)
+  return request_url(url, github_token)
+
+
+def get_last_run_id(
     workflow_id, repo_url, release_branch, release_commit, github_token):
   """
-  Gets data of last run for given repo, branch and commit.
+  Gets id of last run for given workflow, repo, branch and commit.
   Raises exception when no run found.
   """
   url = GH_API_URL_WORKFLOW_RUNS_FMT.format(
@@ -133,10 +137,7 @@ def get_last_run(
       url,
       github_token,
       params={
-          # FIXME change to push
-          "event": "pull_request",
-          "branch": release_branch
-          # "event": "push", "branch": release_branch
+          "event": "push", "branch": release_branch
       },
   )
   runs = safe_get(data, "workflow_runs", url)
@@ -144,11 +145,12 @@ def get_last_run(
   filtered_commit_runs = [
       r for r in runs if r.get("head_sha", "") == release_commit
   ]
+
   if not filtered_commit_runs:
-    workflow_web_url = GH_API_URL_WORKFLOW_RUNS_FMT.format(
+    workflow_run_web_url = GH_API_URL_WORKFLOW_RUNS_FMT.format(
         repo_url=repo_url, workflow_id=workflow_id)
     raise Exception(
-        f"No runs for workflow (branch {release_branch}, commit {release_commit}). Verify at {workflow_web_url}"
+        f"No runs for workflow (branch {release_branch}, commit {release_commit}). Verify at {workflow_run_web_url}"
     )
 
   sorted_runs = sorted(
@@ -160,28 +162,41 @@ def get_last_run(
   print(
       f"Found last run. SHA: {release_commit}, created_at: '{last_run['created_at']}', id: {last_run['id']}"
   )
-  workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
+  workflow_run_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
       repo_url=repo_url, run_id=last_run["id"])
-  print(f"Verify at {workflow_web_url}")
+  print(f"Verify at {workflow_run_web_url}")
   print(
-      f"Upload to GCS available at:\n"
-      f"\tgs://beam-wheels-staging/{release_branch}/{release_commit}-{workflow_id}/"
+      f"Upload to GCS available at: gs://beam-wheels-staging/{release_branch}/{release_commit}-{workflow_id}/"
   )
-  return last_run
+  return safe_get(last_run, "id")
 
 
-def validate_run(run_data, repo_url, github_token):
-  """Validates workflow run. Verifies status and waits if run is not finished."""
+def validate_run(run_id, repo_url, github_token):
+  """Validates workflow run. Verifies succesfull status and waits if run is not finished."""
+  run_data = get_single_workflow_run_data(run_id, repo_url, github_token)
   status = safe_get(run_data, "status")
   conclusion = safe_get(run_data, "conclusion")
+
   if status == "completed" and conclusion == "success":
-    return run_data
+    return run_id
+  elif status in ["queued", "in_progress"]:
+    wait_for_workflow_run_to_finish(
+        run_id, repo_url, status, conclusion, github_token)
+  else:
+    run_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
+        repo_url=repo_url, run_id=run_id)
+    raise Exception(
+        f"Run unsuccessful. Status: {status}. Conclusion: {conclusion}. Check at: {run_web_url}"
+    )
 
-  url = safe_get(run_data, "url")
-  workflow_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
-      repo_url=repo_url, run_id=run_data["id"])
+
+def wait_for_workflow_run_to_finish(
+    run_id, repo_url, status, conclusion, github_token):
+  """Waits for given workflow run to finish succesfully"""
+  run_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
+      repo_url=repo_url, run_id=run_id)
   print(
-      f"Started waiting for Workflow run {run_data['id']} to finish. Check on {workflow_web_url}"
+      f"Started waiting for Workflow run {run_id} to finish. Check on {run_web_url}"
   )
   start_time = time.time()
   last_request = start_time
@@ -200,7 +215,7 @@ def validate_run(run_data, repo_url, github_token):
     time.sleep(0.3)
     if (now - last_request) > request_interval:
       last_request = now
-      run_data = request_url(url, github_token)
+      run_data = get_single_workflow_run_data(run_id, repo_url, github_token)
       status = safe_get(run_data, "status")
       conclusion = safe_get(run_data, "conclusion")
       if status in ["queued", "in_progress"]:
@@ -210,11 +225,12 @@ def validate_run(run_data, repo_url, github_token):
             f"\rFinished in: {elapsed_time}. "
             f"Last state: status: `{status}`, conclusion: `{conclusion}`.",
         )
-        return run_data
+        return run_id
       else:
         print("\r")
         raise Exception(
-            f"Run unsuccessful. Conclusion: {conclusion}. Payload: {run_data}")
+            f"Run unsuccessful. Conclusion: {conclusion}. Check at: {run_web_url}"
+        )
 
 
 def reset_directory(artifacts_dir):
@@ -231,9 +247,11 @@ def reset_directory(artifacts_dir):
     sys.exit(1)
 
 
-def download_artifacts(artifacts_url, artifacts_dir, github_token):
-  """Downloads github artifacts from given url."""
+def download_artifacts(run_id, repo_url, artifacts_dir, github_token):
+  """Downloads github artifacts from given run."""
   print("Starting downloading artifacts ... (it may take a while)")
+  run_data = get_single_workflow_run_data(run_id, repo_url, github_token)
+  artifacts_url = safe_get(run_data, "artifacts_url")
   data_artifacts = request_url(artifacts_url, github_token)
   artifacts = safe_get(data_artifacts, "artifacts", artifacts_url)
   filtered_artifacts = [
@@ -244,21 +262,30 @@ def download_artifacts(artifacts_url, artifacts_dir, github_token):
   for artifact in filtered_artifacts:
     url = safe_get(artifact, "archive_download_url")
     name = safe_get(artifact, "name")
-    artifacts_size_mb = round(artifact["size_in_bytes"] / (1024 * 1024), 2)
-    print(
-        f"\tDownloading {name}.zip artifact (size: {artifacts_size_mb} megabytes)"
-    )
+    size_in_bytes = safe_get(artifact, "size_in_bytes")
+
+    download_single_artifact(
+        url, name, size_in_bytes, artifacts_dir, github_token)
+
+
+def download_single_artifact(
+    url, name, size_in_bytes, artifacts_dir, github_token):
+  """Downloads single github artifact."""
+  artifacts_size_mb = round(size_in_bytes / (1024 * 1024), 2)
+  print(
+      f"\tDownloading {name}.zip artifact (size: {artifacts_size_mb} megabytes)"
+  )
 
-    with tempfile.NamedTemporaryFile(
-        "wb", prefix=name, suffix=".zip"
-    ) as f, request_url(
-        url, github_token, return_json=False, allow_redirects=True, stream=True
-    ) as r:
-      with open(f.name, "wb") as f:
-        shutil.copyfileobj(r.raw, f)
-      with zipfile.ZipFile(f.name, "r") as zip_ref:
-        print(f"\tUnzipping {len(zip_ref.filelist)} files")
-        zip_ref.extractall(artifacts_dir)
+  with tempfile.NamedTemporaryFile(
+    "wb", prefix=name, suffix=".zip"
+  ) as f, request_url(
+    url, github_token, return_json=False, allow_redirects=True, stream=True
+  ) as r:
+    with open(f.name, "wb") as f:
+      shutil.copyfileobj(r.raw, f)
+    with zipfile.ZipFile(f.name, "r") as zip_ref:
+      print(f"\tUnzipping {len(zip_ref.filelist)} files")
+      zip_ref.extractall(artifacts_dir)
 
 
 if __name__ == "__main__":
@@ -276,12 +303,11 @@ def download_artifacts(artifacts_url, artifacts_dir, github_token):
 
   try:
     workflow_id = get_build_wheels_workflow_id(repo_url, github_token)
-    run = get_last_run(
+    run_id = get_last_run_id(
         workflow_id, repo_url, release_branch, release_commit, github_token)
-    run = validate_run(run, repo_url, github_token)
-    artifacts_url = run["artifacts_url"]
+    validate_run(run_id, repo_url, github_token)
     reset_directory(artifacts_dir)
-    download_artifacts(artifacts_url, artifacts_dir, github_token)
+    download_artifacts(run_id, repo_url, artifacts_dir, github_token)
     print("Script finished successfully!")
     print(f"Artifacts available in directory: {artifacts_dir}")
   except KeyboardInterrupt as e:

From 5332c5489a742daa843cb8f7272b0a093fb02ea8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 15 Jul 2020 12:00:42 +0200
Subject: [PATCH 16/35] Add checksum verification of downloaded artifacts

---
 .../main/scripts/build_release_candidate.sh   | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index 05be0434c08f..a5788cf61713 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -19,7 +19,7 @@
 # This script will create a Release Candidate, includes:
 # 1. Build and stage java artifacts
 # 2. Stage source release on dist.apache.org
-# 3. Stage python binaries
+# 3. Stage python binaries and wheels on dist.apache.org
 # 4. Stage SDK docker images
 # 5. Create a PR to update beam-site
 
@@ -195,26 +195,26 @@ if [[ $confirmation = "y" ]]; then
   cp -ar ${PYTHON_ARTIFACTS_DIR}/ beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}/
   cd beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}
 
+  echo "------Checking Hash Value for apache-beam-${RELEASE}.zip-----"
+  sha512sum -c apache-beam-${RELEASE}.zip.sha512
+
   echo "------Signing Source Release apache-beam-${RELEASE}.zip------"
   gpg --local-user ${SIGNING_KEY} --armor --detach-sig apache-beam-${RELEASE}.zip
 
-  echo "------Creating Hash Value for apache-beam-${RELEASE}.zip-----"
-  sha512sum apache-beam-${RELEASE}.zip > apache-beam-${RELEASE}.zip.sha512
+  echo "-----Checking Hash Value for apache-beam-${RELEASE}.tar.gz----"
+  sha512sum -c apache-beam-${RELEASE}.tar.gz.sha512
 
   echo "-----Signing Source Release apache-beam-${RELEASE}.tar.gz-----"
   gpg --local-user ${SIGNING_KEY} --armor --detach-sig apache-beam-${RELEASE}.zip
 
-  echo "-----Creating Hash Value for apache-beam-${RELEASE}.tar.gz----"
-  sha512sum apache-beam-${RELEASE}.zip > apache-beam-${RELEASE}.zip.sha512
-
-  echo "---------Signing Release wheels apache-beam-${RELEASE}--------"
+  echo "-----Checking Hash Value for apache-beam-${RELEASE} wheels-----"
   for artifact in *.whl; do
-    gpg --local-user ${SIGNING_KEY} --armor --detach-sig $artifact
+    sha512sum -c ${artifact}.sha512
   done
 
-  echo "-----Creating Hash Value for apache-beam-${RELEASE} wheels-----"
+  echo "---------Signing Release wheels apache-beam-${RELEASE}--------"
   for artifact in *.whl; do
-    sha512sum $artifact > ${artifact}.sha512
+    gpg --local-user ${SIGNING_KEY} --armor --detach-sig $artifact
   done
 
   cd ..

From ba36b9866003f69e6f115b1bcfec2a3c72b4680f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 15 Jul 2020 13:49:18 +0200
Subject: [PATCH 17/35] fixup! Add checksum verification of downloaded
 artifacts

---
 release/src/main/scripts/build_release_candidate.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index a5788cf61713..8fe62f213c33 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -205,7 +205,7 @@ if [[ $confirmation = "y" ]]; then
   sha512sum -c apache-beam-${RELEASE}.tar.gz.sha512
 
   echo "-----Signing Source Release apache-beam-${RELEASE}.tar.gz-----"
-  gpg --local-user ${SIGNING_KEY} --armor --detach-sig apache-beam-${RELEASE}.zip
+  gpg --local-user ${SIGNING_KEY} --armor --detach-sig apache-beam-${RELEASE}.tar.gz
 
   echo "-----Checking Hash Value for apache-beam-${RELEASE} wheels-----"
   for artifact in *.whl; do

From 94adbc82d255b5be17c5ee4fe28f6c2e78b1ee1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 15 Jul 2020 14:45:10 +0200
Subject: [PATCH 18/35] Improve sh by following shellchecks

---
 .../main/scripts/build_release_candidate.sh   | 50 +++++++++----------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index 8fe62f213c33..69410e4aef22 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -164,57 +164,57 @@ read confirmation
 if [[ $confirmation = "y" ]]; then
   echo "============Staging Python Binaries on dist.apache.org========="
   cd ~
-  if [[ -d ${LOCAL_PYTHON_STAGING_DIR} ]]; then
-    rm -rf ${LOCAL_PYTHON_STAGING_DIR}
+  if [[ -d "${LOCAL_PYTHON_STAGING_DIR}" ]]; then
+    rm -rf "${LOCAL_PYTHON_STAGING_DIR}"
   fi
-  mkdir -p ${LOCAL_PYTHON_STAGING_DIR}
-  cd ${LOCAL_PYTHON_STAGING_DIR}
+  mkdir -p "${LOCAL_PYTHON_STAGING_DIR}"
+  cd "${LOCAL_PYTHON_STAGING_DIR}"
 
   echo '-------------------Cloning Beam Release Branch-----------------'
-  git clone ${GIT_REPO_URL}
-  cd ${BEAM_ROOT_DIR}
-  git checkout ${RELEASE_BRANCH}
+  git clone "${GIT_REPO_URL}"
+  cd "${BEAM_ROOT_DIR}"
+  git checkout "${RELEASE_BRANCH}"
   git push origin "${RELEASE_BRANCH}"
   RELEASE_COMMIT=$(git rev-parse --verify HEAD)
 
   echo '-------------------Creating Python Virtualenv-----------------'
-  python3 -m venv ${LOCAL_PYTHON_VIRTUALENV}
-  source ${LOCAL_PYTHON_VIRTUALENV}/bin/activate
+  python3 -m venv "${LOCAL_PYTHON_VIRTUALENV}"
+  source "${LOCAL_PYTHON_VIRTUALENV}/bin/activate"
   pip install requests python-dateutil
 
   echo '--------------Fetching GitHub Actions Artifacts--------------'
   python release/src/main/scripts/download_github_actions_artifacts.py \
-    --github-user ${USER_GITHUB_ID} \
-    --repo-url ${GIT_REPO_BASE_URL} \
-    --release-branch ${RELEASE_BRANCH} \
-    --release-commit ${RELEASE_COMMIT} \
-    --artifacts_dir ${PYTHON_ARTIFACTS_DIR}
+    --github-user "${USER_GITHUB_ID}" \
+    --repo-url "${GIT_REPO_BASE_URL}" \
+    --release-branch "${RELEASE_BRANCH}" \
+    --release-commit "${RELEASE_COMMIT}" \
+    --artifacts_dir "${PYTHON_ARTIFACTS_DIR}"
 
   svn co https://dist.apache.org/repos/dist/dev/beam
-  mkdir -p beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}
-  cp -ar ${PYTHON_ARTIFACTS_DIR}/ beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}/
-  cd beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}
+  mkdir -p "beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}"
+  cp -ar "${PYTHON_ARTIFACTS_DIR}/" "beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}/"
+  cd "beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}"
 
   echo "------Checking Hash Value for apache-beam-${RELEASE}.zip-----"
-  sha512sum -c apache-beam-${RELEASE}.zip.sha512
+  sha512sum -c "apache-beam-${RELEASE}.zip.sha512"
 
   echo "------Signing Source Release apache-beam-${RELEASE}.zip------"
-  gpg --local-user ${SIGNING_KEY} --armor --detach-sig apache-beam-${RELEASE}.zip
+  gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "apache-beam-${RELEASE}.zip"
 
   echo "-----Checking Hash Value for apache-beam-${RELEASE}.tar.gz----"
-  sha512sum -c apache-beam-${RELEASE}.tar.gz.sha512
+  sha512sum -c "apache-beam-${RELEASE}.tar.gz.sha512"
 
   echo "-----Signing Source Release apache-beam-${RELEASE}.tar.gz-----"
-  gpg --local-user ${SIGNING_KEY} --armor --detach-sig apache-beam-${RELEASE}.tar.gz
+  gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "apache-beam-${RELEASE}.tar.gz"
 
   echo "-----Checking Hash Value for apache-beam-${RELEASE} wheels-----"
   for artifact in *.whl; do
-    sha512sum -c ${artifact}.sha512
+    sha512sum -c "${artifact}.sha512"
   done
 
   echo "---------Signing Release wheels apache-beam-${RELEASE}--------"
   for artifact in *.whl; do
-    gpg --local-user ${SIGNING_KEY} --armor --detach-sig $artifact
+    gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "${artifact}"
   done
 
   cd ..
@@ -224,11 +224,11 @@ if [[ $confirmation = "y" ]]; then
   read confirmation
   if [[ $confirmation != "y" ]]; then
     echo "Exit without staging python artifacts on dist.apache.org."
-    rm -rf ~/${PYTHON_ARTIFACTS_DIR}
+    rm -rf "${HOME:?}/${PYTHON_ARTIFACTS_DIR}"
     exit
   fi
   svn commit --no-auth-cache
-  rm -rf ~/${PYTHON_ARTIFACTS_DIR}
+  rm -rf "${HOME:?}/${PYTHON_ARTIFACTS_DIR}"
 fi
 
 echo "[Current Step]: Stage docker images"

From 4d30bfe84c633be2c6adc551a4048483c4a045bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 21 Jul 2020 10:08:05 +0200
Subject: [PATCH 19/35] Remove remove unnecessary git push

---
 release/src/main/scripts/build_release_candidate.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index 69410e4aef22..1d1ac5bcd2e8 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -174,7 +174,6 @@ if [[ $confirmation = "y" ]]; then
   git clone "${GIT_REPO_URL}"
   cd "${BEAM_ROOT_DIR}"
   git checkout "${RELEASE_BRANCH}"
-  git push origin "${RELEASE_BRANCH}"
   RELEASE_COMMIT=$(git rev-parse --verify HEAD)
 
   echo '-------------------Creating Python Virtualenv-----------------'

From 512a2ad0acd35e119f33db08905ce56b993ab7cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 21 Jul 2020 10:18:31 +0200
Subject: [PATCH 20/35] Move echo inside loop

---
 release/src/main/scripts/build_release_candidate.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index 1d1ac5bcd2e8..f44128b72bc5 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -206,13 +206,13 @@ if [[ $confirmation = "y" ]]; then
   echo "-----Signing Source Release apache-beam-${RELEASE}.tar.gz-----"
   gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "apache-beam-${RELEASE}.tar.gz"
 
-  echo "-----Checking Hash Value for apache-beam-${RELEASE} wheels-----"
   for artifact in *.whl; do
+    echo "----------Checking Hash Value for ${artifact} wheel-----------"
     sha512sum -c "${artifact}.sha512"
   done
 
-  echo "---------Signing Release wheels apache-beam-${RELEASE}--------"
   for artifact in *.whl; do
+    echo "------------------Signing ${artifact} wheel-------------------"
     gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "${artifact}"
   done
 

From 60c50be244bbfaaae227973b2c6eaf303de35523 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 21 Jul 2020 10:20:50 +0200
Subject: [PATCH 21/35] Fix typo

---
 release/src/main/scripts/download_github_actions_artifacts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index db32d619b30d..00a72be99c63 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -83,7 +83,7 @@ def ask_for_github_token():
 
 
 def request_url(url, github_token, return_json=True, *args, **kwargs):
-  """Helper function form making requests authorized by GitHub token."""
+  """Helper function for making requests authorized by GitHub token."""
   r = requests.get(url, *args, auth=("token", github_token), **kwargs)
   if return_json:
     r.raise_for_status()

From 62f69463d70183eee152de694da9a04185d338b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 21 Jul 2020 10:36:25 +0200
Subject: [PATCH 22/35] Fix shadowing variable in context

---
 release/src/main/scripts/download_github_actions_artifacts.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 00a72be99c63..7a3374f87e8e 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -281,8 +281,8 @@ def download_single_artifact(
   ) as f, request_url(
     url, github_token, return_json=False, allow_redirects=True, stream=True
   ) as r:
-    with open(f.name, "wb") as f:
-      shutil.copyfileobj(r.raw, f)
+    with open(f.name, "wb") as _f:
+      shutil.copyfileobj(r.raw, _f)
     with zipfile.ZipFile(f.name, "r") as zip_ref:
       print(f"\tUnzipping {len(zip_ref.filelist)} files")
       zip_ref.extractall(artifacts_dir)

From 44ecec49e0a175c57b7eba15d9a8bb1efcc5c457 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 21 Jul 2020 10:42:01 +0200
Subject: [PATCH 23/35] Improve docstrings

---
 release/src/main/scripts/download_github_actions_artifacts.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 7a3374f87e8e..6f17acf60266 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -92,7 +92,7 @@ def request_url(url, github_token, return_json=True, *args, **kwargs):
 
 
 def safe_get(data, key, url=None):
-  """Gets data by the key with informative Exception in case of non existent key."""
+  """Looks up attribute values from a parsed JSON HTTP response."""
   if key not in data:
     message = f'There is missing key: "{key}" in response data: {data}.'
     if url:
@@ -113,7 +113,7 @@ def get_yes_or_no_answer(question):
 
 
 def get_build_wheels_workflow_id(repo_url, github_token):
-  """Gets workflow id."""
+  """Gets the ID of the Github Actions workflow responsible for building wheels."""
   url = GH_API_URL_WORKLOW_FMT.format(repo_url=repo_url)
   data = request_url(url, github_token)
   return safe_get(data, "id", url)

From 9a02b04401160379f395d7cf62c0c3ad1683ebf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 21 Jul 2020 10:57:07 +0200
Subject: [PATCH 24/35] Fix info about GCS location

---
 .../src/main/scripts/download_github_actions_artifacts.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 6f17acf60266..e9364f221b3a 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -159,14 +159,16 @@ def get_last_run_id(
       reverse=True,
   )
   last_run = sorted_runs[0]
+  last_run_id = safe_get(last_run, "id")
   print(
-      f"Found last run. SHA: {release_commit}, created_at: '{last_run['created_at']}', id: {last_run['id']}"
+      f"Found last run. SHA: {release_commit}, created_at: '{last_run['created_at']}', id: {last_run_id}"
   )
   workflow_run_web_url = GH_WEB_URL_WORKLOW_RUN_FMT.format(
-      repo_url=repo_url, run_id=last_run["id"])
+      repo_url=repo_url, run_id=last_run_id)
   print(f"Verify at {workflow_run_web_url}")
   print(
-      f"Upload to GCS available at: gs://beam-wheels-staging/{release_branch}/{release_commit}-{workflow_id}/"
+      f"GCS location corresponding to artifacts built in this run: "
+      f"gs://beam-wheels-staging/{release_branch}/{release_commit}-{last_run_id}/"
   )
   return safe_get(last_run, "id")
 

From 7ab4468597c47e7b6a1899118475d94efad8ee9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 21 Jul 2020 14:24:12 +0200
Subject: [PATCH 25/35] Refactor fetching GitHub actions artifacts in sh

---
 release/src/main/scripts/build_release_candidate.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index f44128b72bc5..bacced079dce 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -182,17 +182,17 @@ if [[ $confirmation = "y" ]]; then
   pip install requests python-dateutil
 
   echo '--------------Fetching GitHub Actions Artifacts--------------'
+  SVN_ARTIFACTS_DIR="beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}"
+  svn co https://dist.apache.org/repos/dist/dev/beam
+  mkdir -p "${SVN_ARTIFACTS_DIR}"
   python release/src/main/scripts/download_github_actions_artifacts.py \
     --github-user "${USER_GITHUB_ID}" \
     --repo-url "${GIT_REPO_BASE_URL}" \
     --release-branch "${RELEASE_BRANCH}" \
     --release-commit "${RELEASE_COMMIT}" \
-    --artifacts_dir "${PYTHON_ARTIFACTS_DIR}"
+    --artifacts_dir "${SVN_ARTIFACTS_DIR}"
 
-  svn co https://dist.apache.org/repos/dist/dev/beam
-  mkdir -p "beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}"
-  cp -ar "${PYTHON_ARTIFACTS_DIR}/" "beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}/"
-  cd "beam/${RELEASE}/${PYTHON_ARTIFACTS_DIR}"
+  cd "${SVN_ARTIFACTS_DIR}"
 
   echo "------Checking Hash Value for apache-beam-${RELEASE}.zip-----"
   sha512sum -c "apache-beam-${RELEASE}.zip.sha512"

From 9670b6d1456f4082bcfa2b06ba5383d6139da9a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Tue, 21 Jul 2020 14:34:45 +0200
Subject: [PATCH 26/35] Refactor cleaning directory at the end of python
 staging in sh

---
 release/src/main/scripts/build_release_candidate.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index bacced079dce..a561cb5e9600 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -223,11 +223,11 @@ if [[ $confirmation = "y" ]]; then
   read confirmation
   if [[ $confirmation != "y" ]]; then
     echo "Exit without staging python artifacts on dist.apache.org."
-    rm -rf "${HOME:?}/${PYTHON_ARTIFACTS_DIR}"
+    rm -rf "${HOME:?}/${LOCAL_PYTHON_STAGING_DIR}"
     exit
   fi
   svn commit --no-auth-cache
-  rm -rf "${HOME:?}/${PYTHON_ARTIFACTS_DIR}"
+  rm -rf "${HOME:?}/${LOCAL_PYTHON_STAGING_DIR}"
 fi
 
 echo "[Current Step]: Stage docker images"

From 4edf8f0510ec9155df64ac9052ef5aff41d057d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 22 Jul 2020 10:17:53 +0200
Subject: [PATCH 27/35] fixup! Fix info about GCS location

---
 release/src/main/scripts/download_github_actions_artifacts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index e9364f221b3a..66767675db62 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -170,7 +170,7 @@ def get_last_run_id(
       f"GCS location corresponding to artifacts built in this run: "
       f"gs://beam-wheels-staging/{release_branch}/{release_commit}-{last_run_id}/"
   )
-  return safe_get(last_run, "id")
+  return last_run_id
 
 
 def validate_run(run_id, repo_url, github_token):

From 84dbfb0fb1c0f69b9979067cf0228aa9d948c844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 22 Jul 2020 11:37:19 +0200
Subject: [PATCH 28/35] Build and download only ZIP source dist

---
 .github/workflows/build_wheels.yml                    | 11 +++++------
 .../main/scripts/download_github_actions_artifacts.py |  2 +-
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index 73476cb913f4..6737a1996806 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -52,13 +52,12 @@ jobs:
         run: python -m pip install wheel
       - name: Build source
         working-directory: ./sdks/python
-        run: python setup.py sdist --formats=gztar,zip
+        run: python setup.py sdist --formats=zip
       - name: Add checksums
         working-directory: ./sdks/python/dist
         run: |
-          for file in *.zip *.tar.gz; do
-            sha512sum $file > ${file}.sha512
-          done
+          file=$(ls | grep .zip | head -n 1)
+          sha512sum $file > ${file}.sha512
       - name: Unzip source
         working-directory: ./sdks/python
         run: unzip dist/$(ls dist | grep .zip | head -n 1)
@@ -73,7 +72,7 @@ jobs:
       - name: Upload compressed sources as artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: source_gztar_zip
+          name: source_zip
           path: sdks/python/dist
 
   prepare_gcs:
@@ -99,7 +98,7 @@ jobs:
       - name: Download compressed sources from artifacts
         uses: actions/download-artifact@v2
         with:
-          name: source_gztar_zip
+          name: source_zip
           path: source/
       - name: Authenticate on GCP
         uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 66767675db62..91a4336fb78f 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -258,7 +258,7 @@ def download_artifacts(run_id, repo_url, artifacts_dir, github_token):
   artifacts = safe_get(data_artifacts, "artifacts", artifacts_url)
   filtered_artifacts = [
       a for a in artifacts if (
-          a["name"].startswith("source_gztar_zip") or
+          a["name"].startswith("source_zip") or
           a["name"].startswith("wheelhouse"))
   ]
   for artifact in filtered_artifacts:

From 49e8084ba057b0b7ad4bf76c0ac009f999460378 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 22 Jul 2020 12:46:38 +0200
Subject: [PATCH 29/35] Refactor downloading artifacts

---
 .../download_github_actions_artifacts.py      | 42 +++++++++++--------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 91a4336fb78f..a67a317a2335 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -249,8 +249,8 @@ def reset_directory(artifacts_dir):
     sys.exit(1)
 
 
-def download_artifacts(run_id, repo_url, artifacts_dir, github_token):
-  """Downloads github artifacts from given run."""
+def fetch_github_artifacts(run_id, repo_url, artifacts_dir, github_token):
+  """Downloads and extracts github artifacts with source dist and wheels from given run."""
   print("Starting downloading artifacts ... (it may take a while)")
   run_data = get_single_workflow_run_data(run_id, repo_url, github_token)
   artifacts_url = safe_get(run_data, "artifacts_url")
@@ -266,28 +266,36 @@ def download_artifacts(run_id, repo_url, artifacts_dir, github_token):
     name = safe_get(artifact, "name")
     size_in_bytes = safe_get(artifact, "size_in_bytes")
 
-    download_single_artifact(
-        url, name, size_in_bytes, artifacts_dir, github_token)
+    fd, temp_file_path = tempfile.mkstemp(prefix=name, suffix=".zip")
+    try:
+      os.close(fd)
+      download_single_artifact(
+          url, name, size_in_bytes, temp_file_path, github_token)
+      extract_single_artifact(temp_file_path, artifacts_dir)
+    finally:
+      os.remove(temp_file_path)
 
 
 def download_single_artifact(
-    url, name, size_in_bytes, artifacts_dir, github_token):
-  """Downloads single github artifact."""
+    url, name, size_in_bytes, target_file_path, github_token):
   artifacts_size_mb = round(size_in_bytes / (1024 * 1024), 2)
   print(
       f"\tDownloading {name}.zip artifact (size: {artifacts_size_mb} megabytes)"
   )
 
-  with tempfile.NamedTemporaryFile(
-    "wb", prefix=name, suffix=".zip"
-  ) as f, request_url(
-    url, github_token, return_json=False, allow_redirects=True, stream=True
-  ) as r:
-    with open(f.name, "wb") as _f:
-      shutil.copyfileobj(r.raw, _f)
-    with zipfile.ZipFile(f.name, "r") as zip_ref:
-      print(f"\tUnzipping {len(zip_ref.filelist)} files")
-      zip_ref.extractall(artifacts_dir)
+  with request_url(url,
+                   github_token,
+                   return_json=False,
+                   allow_redirects=True,
+                   stream=True) as r:
+    with open(target_file_path, "wb") as f:
+      shutil.copyfileobj(r.raw, f)
+
+
+def extract_single_artifact(file_path, output_dir):
+  with zipfile.ZipFile(file_path, "r") as zip_ref:
+    print(f"\tUnzipping {len(zip_ref.filelist)} files")
+    zip_ref.extractall(output_dir)
 
 
 if __name__ == "__main__":
@@ -309,7 +317,7 @@ def download_single_artifact(
         workflow_id, repo_url, release_branch, release_commit, github_token)
     validate_run(run_id, repo_url, github_token)
     reset_directory(artifacts_dir)
-    download_artifacts(run_id, repo_url, artifacts_dir, github_token)
+    fetch_github_artifacts(run_id, repo_url, artifacts_dir, github_token)
     print("Script finished successfully!")
     print(f"Artifacts available in directory: {artifacts_dir}")
   except KeyboardInterrupt as e:

From a67033e175cefe3b8e1068e1b40837d78881a5ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Wed, 22 Jul 2020 12:49:05 +0200
Subject: [PATCH 30/35] fixup! Build and download only ZIP source dist

---
 release/src/main/scripts/build_release_candidate.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index a561cb5e9600..d17c94938713 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -200,12 +200,6 @@ if [[ $confirmation = "y" ]]; then
   echo "------Signing Source Release apache-beam-${RELEASE}.zip------"
   gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "apache-beam-${RELEASE}.zip"
 
-  echo "-----Checking Hash Value for apache-beam-${RELEASE}.tar.gz----"
-  sha512sum -c "apache-beam-${RELEASE}.tar.gz.sha512"
-
-  echo "-----Signing Source Release apache-beam-${RELEASE}.tar.gz-----"
-  gpg --local-user "${SIGNING_KEY}" --armor --detach-sig "apache-beam-${RELEASE}.tar.gz"
-
   for artifact in *.whl; do
     echo "----------Checking Hash Value for ${artifact} wheel-----------"
     sha512sum -c "${artifact}.sha512"

From a8fed686321235b19cbadae4d6b7baec92a0d34c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Sat, 25 Jul 2020 19:33:56 +0200
Subject: [PATCH 31/35] Improve handling temporary files

---
 .../src/main/scripts/download_github_actions_artifacts.py  | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index a67a317a2335..b4e3d9991d44 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -266,14 +266,11 @@ def fetch_github_artifacts(run_id, repo_url, artifacts_dir, github_token):
     name = safe_get(artifact, "name")
     size_in_bytes = safe_get(artifact, "size_in_bytes")
 
-    fd, temp_file_path = tempfile.mkstemp(prefix=name, suffix=".zip")
-    try:
-      os.close(fd)
+    with tempfile.TemporaryDirectory() as tmp:
+      temp_file_path = os.path.join(tmp, name + ".zip")
       download_single_artifact(
           url, name, size_in_bytes, temp_file_path, github_token)
       extract_single_artifact(temp_file_path, artifacts_dir)
-    finally:
-      os.remove(temp_file_path)
 
 
 def download_single_artifact(

From 54766cec7644ec6513045876028d9b59eb4df31b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Sat, 25 Jul 2020 19:55:17 +0200
Subject: [PATCH 32/35] Use absolute path for artifacts directory

---
 release/src/main/scripts/download_github_actions_artifacts.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index b4e3d9991d44..08c18a812181 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -64,7 +64,8 @@ def parse_arguments():
   repo_url = args.repo_url
   release_branch = args.release_branch
   release_commit = args.release_commit
-  artifacts_dir = args.artifacts_dir
+  artifacts_dir = args.artifacts_dir if os.path.isabs(args.artifacts_dir) \
+    else os.path.abspath(args.artifacts_dir)
 
   return github_token, user_github_id, repo_url, release_branch, release_commit, artifacts_dir
 

From fe4f834d3142c26cc216c90014cf364f24dbaf47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Sat, 25 Jul 2020 20:14:38 +0200
Subject: [PATCH 33/35] Move pre-requirements to the top

---
 release/src/main/scripts/build_release_candidate.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index d17c94938713..0d4885a29209 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -63,6 +63,8 @@ read USER_GITHUB_ID
 
 USER_REMOTE_URL=git@github.com:${USER_GITHUB_ID}/beam-site
 
+echo "=================Pre-requirements===================="
+echo "Please make sure you have configured and started your gpg by running ./preparation_before_release.sh."
 echo "================Listing all GPG keys================="
 gpg --list-keys --keyid-format LONG --fingerprint --fingerprint
 echo "Please copy the public key which is associated with your Apache account:"
@@ -157,8 +159,6 @@ fi
 
 
 echo "[Current Step]: Stage python binaries and wheels"
-echo "===============================Pre-requirements========================"
-echo "Please make sure you have configured and started your gpg by running ./preparation_before_release.sh."
 echo "Do you want to proceed? [y|N]"
 read confirmation
 if [[ $confirmation = "y" ]]; then

From 6abb237372972d17f9a460ee76435718f241a145 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Sat, 25 Jul 2020 20:34:03 +0200
Subject: [PATCH 34/35] Improve description

---
 release/src/main/scripts/build_release_candidate.sh     | 4 ++--
 website/www/site/content/en/contribute/release-guide.md | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
index 0d4885a29209..90eed1afeb3f 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -19,7 +19,7 @@
 # This script will create a Release Candidate, includes:
 # 1. Build and stage java artifacts
 # 2. Stage source release on dist.apache.org
-# 3. Stage python binaries and wheels on dist.apache.org
+# 3. Stage python source distribution and wheels on dist.apache.org
 # 4. Stage SDK docker images
 # 5. Create a PR to update beam-site
 
@@ -158,7 +158,7 @@ if [[ $confirmation = "y" ]]; then
 fi
 
 
-echo "[Current Step]: Stage python binaries and wheels"
+echo "[Current Step]: Stage python source distribution and wheels on dist.apache.org"
 echo "Do you want to proceed? [y|N]"
 read confirmation
 if [[ $confirmation = "y" ]]; then
diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md
index 845cda0c1e5f..5768ae7c9379 100644
--- a/website/www/site/content/en/contribute/release-guide.md
+++ b/website/www/site/content/en/contribute/release-guide.md
@@ -562,7 +562,7 @@ For this step, we recommend you using automation script to create a RC, but you
   1. Run gradle release to create rc tag and push source release into github repo.
   1. Run gradle publish to push java artifacts into Maven staging repo.
   1. Stage source release into dist.apache.org dev [repo](https://dist.apache.org/repos/dist/dev/beam/).
-  1. Stage, sign and hash python binaries and wheels into dist.apache.ord dev repo python dir
+  1. Stage, sign and hash python source distribution and wheels into dist.apache.org dev repo python dir
   1. Stage SDK docker images to [docker hub Apache organization](https://hub.docker.com/search?q=apache%2Fbeam&type=image).
   1. Create a PR to update beam-site, changes includes:
      * Copy python doc into beam-site

From 54e61392ad7407b6e534ff5f7799edd86f026ace Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobiasz=20K=C4=99dzierski?= <tobiasz.kedzierski@polidea.com>
Date: Sat, 25 Jul 2020 20:57:08 +0200
Subject: [PATCH 35/35] Ask for clearing dir confirmation if dir exists

---
 .../download_github_actions_artifacts.py      | 30 +++++++++++--------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/release/src/main/scripts/download_github_actions_artifacts.py b/release/src/main/scripts/download_github_actions_artifacts.py
index 08c18a812181..3fdbb282411b 100644
--- a/release/src/main/scripts/download_github_actions_artifacts.py
+++ b/release/src/main/scripts/download_github_actions_artifacts.py
@@ -236,18 +236,22 @@ def wait_for_workflow_run_to_finish(
         )
 
 
-def reset_directory(artifacts_dir):
-  """Clears directory asking for confirmation before."""
-  question = (
-      f"Creating Artifacts directory. Any existing content in {artifacts_dir} will be erased. Proceed?\n"
-      f"Your answer")
-  if get_yes_or_no_answer(question):
-    print(f"Clearing directory: {artifacts_dir}")
-    shutil.rmtree(artifacts_dir, ignore_errors=True)
-    os.makedirs(artifacts_dir)
-  else:
-    print("You said NO for clearing artifacts directory. Quitting ...")
-    sys.exit(1)
+def prepare_directory(artifacts_dir):
+  """Creates given directory and asks for confirmation if directory exists before clearing it."""
+  print(f"Preparing Artifacts directory: {artifacts_dir}")
+  if os.path.isdir(artifacts_dir):
+    question = (
+        f"Found that directory already exists.\n"
+        f"Any existing content in it will be erased. Proceed?\n"
+        f"Your answer")
+    if get_yes_or_no_answer(question):
+      print(f"Clearing directory: {artifacts_dir}")
+      shutil.rmtree(artifacts_dir, ignore_errors=True)
+    else:
+      print("You said NO for clearing artifacts directory. Quitting ...")
+      sys.exit(1)
+
+  os.makedirs(artifacts_dir)
 
 
 def fetch_github_artifacts(run_id, repo_url, artifacts_dir, github_token):
@@ -314,7 +318,7 @@ def extract_single_artifact(file_path, output_dir):
     run_id = get_last_run_id(
         workflow_id, repo_url, release_branch, release_commit, github_token)
     validate_run(run_id, repo_url, github_token)
-    reset_directory(artifacts_dir)
+    prepare_directory(artifacts_dir)
     fetch_github_artifacts(run_id, repo_url, artifacts_dir, github_token)
     print("Script finished successfully!")
     print(f"Artifacts available in directory: {artifacts_dir}")