diff --git a/.asf.yaml b/.asf.yaml index f478c61fe3e39..65a3314544812 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -50,7 +50,6 @@ github: # See ./github/workflows/README.md for more documentation on this list. contexts: - Pulsar CI checks completed - - cpp-tests required_pull_request_reviews: dismiss_stale_reviews: false diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml deleted file mode 100644 index 7b98ff9309aa7..0000000000000 --- a/.github/workflows/ci-cpp.yaml +++ /dev/null @@ -1,104 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - CPP, Python Tests -on: - pull_request: - branches: - - master - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - cpp-tests: - name: - runs-on: ubuntu-20.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - list-files: csv - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 17 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: build package - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests - - - name: build cpp artifacts - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - echo "Build C++ client library" - pulsar-client-cpp/docker-build.sh - - - name: run c++ tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: pulsar-client-cpp/docker-tests.sh - - - name: Upload test-logs - uses: actions/upload-artifact@v3 - if: failure() - continue-on-error: true - with: - name: test-logs - path: test-logs diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 1cc95c1f56cd4..51f98aee39088 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -115,6 +115,9 @@ jobs: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \ tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \ -I zstd -cf - .m2/repository/org/apache/pulsar + cd $GITHUB_WORKSPACE + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \ + tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz - name: Wait for ssh connection when build fails # ssh access is enabled for builds in own forks @@ -125,7 +128,6 @@ jobs: action: wait - unit-tests: name: CI - Unit - ${{ matrix.name }} runs-on: ubuntu-20.04 @@ -466,6 +468,71 @@ jobs: run: | gh-actions-artifact-client.js delete pulsar-java-test-image.zst + cpp-tests: + name: + runs-on: ubuntu-20.04 + timeout-minutes: 120 + needs: [ + 'changed_files_job', + 'integration-tests' + ] + if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: 17 + + - name: Clean Disk + uses: ./.github/actions/clean-disk + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + cd $GITHUB_WORKSPACE + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-server-distribution + + - name: copy python tests + run: | + mvn -B -Pskip-all -ntp -pl pulsar-functions/instance package + + - name: build cpp artifacts + run: | + echo "Build C++ client library" + pulsar-client-cpp/docker-build.sh + + - name: run c++ tests + run: pulsar-client-cpp/docker-tests.sh + + - name: Upload test-logs + uses: actions/upload-artifact@v3 + if: failure() + continue-on-error: true + with: + name: cpp-tests-logs + path: test-logs + pulsar-test-latest-version-image: name: Build Pulsar docker image runs-on: ubuntu-20.04 @@ -822,7 +889,7 @@ jobs: - name: Delete docker image from GitHub Actions Artifacts run: | gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst - + macos-build: name: Build Pulsar on MacOS runs-on: macos-11 @@ -890,13 +957,9 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.jdk || '17' }} - - name: clean disk - run: | - sudo swapoff -a - sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h + - name: Clean Disk + uses: ./.github/actions/clean-disk + - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -938,7 +1001,8 @@ jobs: 'integration-tests', 'system-tests', 'flaky-system-tests', - 'macos-build' + 'macos-build', + 'cpp-tests' ] steps: - name: Check that all required jobs were completed successfully @@ -949,6 +1013,7 @@ jobs: && "${{ needs.integration-tests.result }}" == "success" \ && "${{ needs.system-tests.result }}" == "success" \ && "${{ needs.macos-build.result }}" == "success" \ + && "${{ needs.cpp-tests.result }}" == "success" \ ) ]]; then echo "Required jobs haven't been completed successfully." exit 1 @@ -969,4 +1034,5 @@ jobs: - name: Delete maven repository binaries from GitHub Actions Artifacts if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} run: | - gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst + gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true + gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true diff --git a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh index 613e7b4db220b..5cb729686790f 100755 --- a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh +++ b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh @@ -25,7 +25,7 @@ pip3 install protobuf==3.20.1 --user pip3 install fastavro --user CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -PULSAR_HOME=$CUR_DIR/../../../../ +PULSAR_HOME="$( cd "$CUR_DIR/../../../../" >/dev/null && pwd )" # run instance tests PULSAR_HOME=${PULSAR_HOME} PYTHONPATH=${PULSAR_HOME}/pulsar-functions/instance/target/python-instance python3 -m unittest discover -v ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests