From 7bc2ab7c234bd4992924ffe5dae64a561136a9a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 9 Sep 2022 11:25:27 +0200 Subject: [PATCH 1/8] [ci] Move cpp-tests job inside Pulsar CI --- .asf.yaml | 1 - .github/workflows/ci-cpp.yaml | 104 ------------------------------- .github/workflows/pulsar-ci.yaml | 67 +++++++++++++++++++- 3 files changed, 65 insertions(+), 107 deletions(-) delete mode 100644 .github/workflows/ci-cpp.yaml diff --git a/.asf.yaml b/.asf.yaml index f478c61fe3e39..65a3314544812 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -50,7 +50,6 @@ github: # See ./github/workflows/README.md for more documentation on this list. contexts: - Pulsar CI checks completed - - cpp-tests required_pull_request_reviews: dismiss_stale_reviews: false diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml deleted file mode 100644 index 7b98ff9309aa7..0000000000000 --- a/.github/workflows/ci-cpp.yaml +++ /dev/null @@ -1,104 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - CPP, Python Tests -on: - pull_request: - branches: - - master - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - cpp-tests: - name: - runs-on: ubuntu-20.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 17 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: build package - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests - - - name: build cpp artifacts - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - echo "Build C++ client library" - pulsar-client-cpp/docker-build.sh - - - name: run c++ tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: pulsar-client-cpp/docker-tests.sh - - - name: Upload test-logs - uses: actions/upload-artifact@v3 - if: failure() - continue-on-error: true - with: - name: test-logs - path: test-logs diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 1cc95c1f56cd4..13c3c6e2ab690 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -125,7 +125,6 @@ jobs: action: wait - unit-tests: name: CI - Unit - ${{ matrix.name }} runs-on: ubuntu-20.04 @@ -466,6 +465,68 @@ jobs: run: | gh-actions-artifact-client.js delete pulsar-java-test-image.zst + cpp-tests: + name: + runs-on: ubuntu-20.04 + timeout-minutes: 120 + needs: [ + 'changed_files_job', + 'integration-tests' + ] + if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: 17 + + - name: clean disk + run: | + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: build cpp artifacts + run: | + echo "Build C++ client library" + pulsar-client-cpp/docker-build.sh + + - name: run c++ tests + run: pulsar-client-cpp/docker-tests.sh + + - name: Upload test-logs + uses: actions/upload-artifact@v3 + if: failure() + continue-on-error: true + with: + name: cpp-tests-logs + path: test-logs + pulsar-test-latest-version-image: name: Build Pulsar docker image runs-on: ubuntu-20.04 @@ -938,7 +999,8 @@ jobs: 'integration-tests', 'system-tests', 'flaky-system-tests', - 'macos-build' + 'macos-build', + 'cpp-tests' ] steps: - name: Check that all required jobs were completed successfully @@ -949,6 +1011,7 @@ jobs: && "${{ needs.integration-tests.result }}" == "success" \ && "${{ needs.system-tests.result }}" == "success" \ && "${{ needs.macos-build.result }}" == "success" \ + && "${{ needs.cpp-tests.result }}" == "success" \ ) ]]; then echo "Required jobs haven't been completed successfully." exit 1 From 2206b1b1ccad9149620c63110e6d956184986e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 12 Sep 2022 15:45:43 +0200 Subject: [PATCH 2/8] move to common clean disk action --- .github/workflows/pulsar-ci.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 13c3c6e2ab690..e63a803caec18 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -497,11 +497,8 @@ jobs: distribution: 'temurin' java-version: 17 - - name: clean disk - run: | - sudo apt clean - docker rmi $(docker images -q) -f - df -h + - name: Clean Disk + uses: ./.github/actions/clean-disk - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master From 319b19b991bd8beacc8f2914f36704ca91cea4d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 12 Sep 2022 16:00:29 +0200 Subject: [PATCH 3/8] use clean-disk in owasp step --- .github/workflows/pulsar-ci.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index e63a803caec18..39bb6986da9e7 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -948,13 +948,9 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.jdk || '17' }} - - name: clean disk - run: | - sudo swapoff -a - sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h + - name: Clean Disk + uses: ./.github/actions/clean-disk + - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master From aba24dbe163b63def665d834d687c10600ffd76e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 12 Sep 2022 18:21:15 +0300 Subject: [PATCH 4/8] Cache pulsar-server-distribution for cpp-tests --- .github/workflows/pulsar-ci.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 39bb6986da9e7..481a0c9d56e8c 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -115,6 +115,9 @@ jobs: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \ tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \ -I zstd -cf - .m2/repository/org/apache/pulsar + cd $GITHUB_WORKSPACE + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \ + tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks @@ -507,6 +510,8 @@ jobs: run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + cd $GITHUB_WORKSPACE + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-server-distribution - name: build cpp artifacts run: | @@ -880,7 +885,7 @@ jobs: - name: Delete docker image from GitHub Actions Artifacts run: | gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst - + macos-build: name: Build Pulsar on MacOS runs-on: macos-11 @@ -1025,4 +1030,5 @@ jobs: - name: Delete maven repository binaries from GitHub Actions Artifacts if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} run: | - gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst + gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true + gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true From 9830ed5c533452bef32459f94fdc2f2d3dfe9068 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 12 Sep 2022 19:53:59 +0300 Subject: [PATCH 5/8] cleanup PULSAR_HOME --- .../instance/src/scripts/run_python_instance_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh index 613e7b4db220b..5cb729686790f 100755 --- a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh +++ b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh @@ -25,7 +25,7 @@ pip3 install protobuf==3.20.1 --user pip3 install fastavro --user CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -PULSAR_HOME=$CUR_DIR/../../../../ +PULSAR_HOME="$( cd "$CUR_DIR/../../../../" >/dev/null && pwd )" # run instance tests PULSAR_HOME=${PULSAR_HOME} PYTHONPATH=${PULSAR_HOME}/pulsar-functions/instance/target/python-instance python3 -m unittest discover -v ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests From be7a5cfb34f7e188da80a2350d47498f7c4e9533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 12 Sep 2022 21:49:01 +0200 Subject: [PATCH 6/8] add python debug --- .../instance/src/scripts/run_python_instance_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh index 5cb729686790f..040c83f561e42 100755 --- a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh +++ b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh @@ -27,5 +27,7 @@ pip3 install fastavro --user CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" PULSAR_HOME="$( cd "$CUR_DIR/../../../../" >/dev/null && pwd )" +ls -la ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests || echo "dir not found" +touch ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests/__init__.py || echo "dir not found 2" # run instance tests -PULSAR_HOME=${PULSAR_HOME} PYTHONPATH=${PULSAR_HOME}/pulsar-functions/instance/target/python-instance python3 -m unittest discover -v ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests +PULSAR_HOME=${PULSAR_HOME} PYTHONPATH=${PULSAR_HOME}/pulsar-functions/instance/target/python-instance python3 -m unittest discover -v -s ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests From cb107ab8e283f8897fe3c1b740c48c67c0749c47 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 13 Sep 2022 06:45:47 +0300 Subject: [PATCH 7/8] Revert "add python debug" This reverts commit be7a5cfb34f7e188da80a2350d47498f7c4e9533. --- .../instance/src/scripts/run_python_instance_tests.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh index 040c83f561e42..5cb729686790f 100755 --- a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh +++ b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh @@ -27,7 +27,5 @@ pip3 install fastavro --user CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" PULSAR_HOME="$( cd "$CUR_DIR/../../../../" >/dev/null && pwd )" -ls -la ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests || echo "dir not found" -touch ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests/__init__.py || echo "dir not found 2" # run instance tests -PULSAR_HOME=${PULSAR_HOME} PYTHONPATH=${PULSAR_HOME}/pulsar-functions/instance/target/python-instance python3 -m unittest discover -v -s ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests +PULSAR_HOME=${PULSAR_HOME} PYTHONPATH=${PULSAR_HOME}/pulsar-functions/instance/target/python-instance python3 -m unittest discover -v ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests From 102c944a9cc99965270b6a0185e63c4a60a94b2e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 13 Sep 2022 07:03:47 +0300 Subject: [PATCH 8/8] Copy python tests --- .github/workflows/pulsar-ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 481a0c9d56e8c..51f98aee39088 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -513,6 +513,10 @@ jobs: cd $GITHUB_WORKSPACE $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-server-distribution + - name: copy python tests + run: | + mvn -B -Pskip-all -ntp -pl pulsar-functions/instance package + - name: build cpp artifacts run: | echo "Build C++ client library"