diff --git a/.github/actions/upload-coverage/action.yml b/.github/actions/upload-coverage/action.yml index faa4d510350d2..97d311caf5016 100644 --- a/.github/actions/upload-coverage/action.yml +++ b/.github/actions/upload-coverage/action.yml @@ -57,6 +57,10 @@ runs: flags: ${{ inputs.flags }} fail_ci_if_error: true verbose: true + - name: "Wait 15 seconds before next attempt" + if: steps.codecov-upload-1.outcome == 'failure' + shell: bash + run: sleep 15 - name: "Upload to Codecov (attempt #2)" id: codecov-upload-2 if: steps.codecov-upload-1.outcome == 'failure' @@ -66,11 +70,16 @@ runs: flags: ${{ inputs.flags }} fail_ci_if_error: true verbose: true + - name: "Wait 60 seconds before next attempt" + if: steps.codecov-upload-2.outcome == 'failure' + shell: bash + run: sleep 60 - name: "Upload to Codecov (attempt #3)" id: codecov-upload-3 if: steps.codecov-upload-2.outcome == 'failure' uses: codecov/codecov-action@v3 - continue-on-error: true + # fail on last attempt + continue-on-error: false with: flags: ${{ inputs.flags }} fail_ci_if_error: true diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index 1b17e3ecbc6f2..c47dfcf9c738f 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -127,6 +127,19 @@ jobs: if: ${{ always() }} uses: ./.github/actions/copy-test-reports + - name: Create Jacoco reports + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -r jacoco_test_coverage_report_flaky.zip jacoco_test_coverage_report + + - name: Upload Jacoco report files to build artifacts + uses: actions/upload-artifact@v3 + with: + name: Jacoco-coverage-report-flaky + path: target/jacoco_test_coverage_report_flaky.zip + retention-days: 3 + - name: Upload to Codecov uses: ./.github/actions/upload-coverage with: diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index abb015ed15c8b..2c55b28cbc538 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -229,6 +229,7 @@ jobs: run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - name: Run setup commands run: | @@ -238,6 +239,9 @@ jobs: run: | CHANGED_TESTS="${{ needs.preconditions.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }} + - name: Upload coverage to build artifacts + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_unittest_coverage_files ${{ matrix.group }} + - name: print JVM thread dumps when cancelled if: cancelled() run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps @@ -246,11 +250,6 @@ jobs: if: ${{ always() }} uses: ./.github/actions/copy-test-reports - - name: Upload to Codecov - uses: ./.github/actions/upload-coverage - with: - flags: unittests - - name: Publish Test Report uses: apache/pulsar-test-infra/action-junit-report@master if: ${{ always() }} @@ -286,6 +285,86 @@ jobs: with: action: wait + + unit-tests-upload-coverage: + name: CI - Unit - Upload Coverage + runs-on: ubuntu-20.04 + timeout-minutes: 30 + needs: ['unit-tests'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK ${{ matrix.jdk || '17' }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk || '17' }} + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Restore coverage files from build artifacts and create Jacoco reports + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -r jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report + + - name: Upload Jacoco report files to build artifacts + uses: actions/upload-artifact@v3 + with: + name: Jacoco-coverage-report-unittests + path: target/jacoco_test_coverage_report_unittests.zip + retention-days: 3 + + - name: Upload to Codecov + uses: ./.github/actions/upload-coverage + with: + flags: unittests + + - name: Delete coverage files from build artifacts + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + pulsar-java-test-image: name: Build Pulsar java-test-image docker image runs-on: ubuntu-20.04 @@ -451,6 +530,7 @@ jobs: run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache run: | @@ -475,10 +555,8 @@ jobs: fi ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args - - name: Upload to Codecov - uses: ./.github/actions/upload-coverage - with: - flags: inttests + - name: Upload coverage to build artifacts + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }} - name: print JVM thread dumps when cancelled if: cancelled() @@ -520,13 +598,99 @@ jobs: with: action: wait + integration-tests-upload-coverage: + name: CI - Integration - Upload Coverage + runs-on: ubuntu-20.04 + timeout-minutes: 30 + needs: ['integration-tests'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts + + - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image + + - name: Restore coverage files from build artifacts and create Jacoco reports + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -r jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report + + - name: Upload Jacoco report files to build artifacts + uses: actions/upload-artifact@v3 + with: + name: Jacoco-coverage-report-inttests + path: target/jacoco_test_coverage_report_inttests.zip + retention-days: 3 + + - name: Upload to Codecov + uses: ./.github/actions/upload-coverage + with: + flags: inttests + + - name: Delete coverage files from build artifacts + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + delete-integration-test-docker-image-artifact: name: "Delete integration test docker image artifact" runs-on: ubuntu-20.04 timeout-minutes: 10 needs: [ 'preconditions', - 'integration-tests' + 'integration-tests', + 'integration-tests-upload-coverage' ] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: @@ -738,6 +902,7 @@ jobs: run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache run: | @@ -752,10 +917,8 @@ jobs: run: | ./build/run_integration_group.sh ${{ matrix.group }} --coverage - - name: Upload to Codecov - uses: ./.github/actions/upload-coverage - with: - flags: systests + - name: Upload coverage to build artifacts + run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }} - name: print JVM thread dumps when cancelled if: cancelled() @@ -797,6 +960,92 @@ jobs: with: action: wait + system-tests-upload-coverage: + name: CI - System - Upload Coverage + runs-on: ubuntu-20.04 + timeout-minutes: 30 + needs: ['system-tests'] + if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + uses: ./.github/actions/ssh-access + continue-on-error: true + with: + limit-access-to-actor: true + + - name: Cache local Maven repository + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 17 + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image + + - name: Restore coverage files from build artifacts and create Jacoco reports + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report + cd $GITHUB_WORKSPACE/target + zip -r jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report + + - name: Upload Jacoco report files to build artifacts + uses: actions/upload-artifact@v3 + with: + name: Jacoco-coverage-report-systests + path: target/jacoco_test_coverage_report_systests.zip + retention-days: 3 + + - name: Upload to Codecov + uses: ./.github/actions/upload-coverage + with: + flags: systests + + - name: Delete coverage files from build artifacts + run: | + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} + continue-on-error: true + with: + action: wait + flaky-system-tests: name: CI Flaky - System - ${{ matrix.name }} runs-on: ubuntu-20.04 @@ -816,7 +1065,6 @@ jobs: - name: Pulsar IO - Oracle group: PULSAR_IO_ORA - steps: - name: checkout uses: actions/checkout@v3 @@ -923,6 +1171,7 @@ jobs: needs: [ 'preconditions', 'system-tests', + 'system-tests-upload-coverage', 'flaky-system-tests' ] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} @@ -1069,7 +1318,10 @@ jobs: 'integration-tests', 'system-tests', 'flaky-system-tests', - 'macos-build' + 'macos-build', + 'unit-tests-upload-coverage', + 'integration-tests-upload-coverage', + 'system-tests-upload-coverage' ] steps: - name: Check that all required jobs were completed successfully diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh index 31bb18f746f93..613b58bcb582b 100755 --- a/build/pulsar_ci_tool.sh +++ b/build/pulsar_ci_tool.sh @@ -20,6 +20,8 @@ # shell function library for Pulsar CI builds +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + set -e set -o pipefail @@ -91,14 +93,18 @@ function ci_install_tool() { local tool_executable=$1 local tool_package=${2:-$1} if ! command -v $tool_executable &>/dev/null; then - echo "::group::Installing ${tool_package}" - sudo apt-get -y install ${tool_package} >/dev/null || { - echo "Installing the package failed. Switching the ubuntu mirror and retrying..." - ci_pick_ubuntu_mirror - # retry after picking the ubuntu mirror - sudo apt-get -y install ${tool_package} - } - echo '::endgroup::' + if [[ "$GITHUB_ACTIONS" == "true" ]]; then + echo "::group::Installing ${tool_package}" + sudo apt-get -y install ${tool_package} >/dev/null || { + echo "Installing the package failed. Switching the ubuntu mirror and retrying..." + ci_pick_ubuntu_mirror + # retry after picking the ubuntu mirror + sudo apt-get -y install ${tool_package} + } + echo '::endgroup::' + else + fail "$tool_executable wasn't found on PATH. You should first install $tool_package with your package manager." + fi fi } @@ -115,8 +121,8 @@ function ci_docker_save_image_to_github_actions_artifacts() { ci_install_tool pv echo "::group::Saving docker image ${image} with name ${artifactname} in GitHub Actions Artifacts" # delete possible previous artifact that might exist when re-running - gh-actions-artifact-client.js delete "${artifactname}" &>/dev/null || true - docker save ${image} | zstd | pv -ft -i 5 | pv -Wbaf -i 5 | gh-actions-artifact-client.js upload --retentionDays=$ARTIFACT_RETENTION_DAYS "${artifactname}" + timeout 1m gh-actions-artifact-client.js delete "${artifactname}" &>/dev/null || true + docker save ${image} | zstd | pv -ft -i 5 | pv -Wbaf -i 5 | timeout 20m gh-actions-artifact-client.js upload --retentionDays=$ARTIFACT_RETENTION_DAYS "${artifactname}" echo "::endgroup::" } @@ -125,7 +131,7 @@ function ci_docker_load_image_from_github_actions_artifacts() { local artifactname="${1}.zst" ci_install_tool pv echo "::group::Loading docker image from name ${artifactname} in GitHub Actions Artifacts" - gh-actions-artifact-client.js download "${artifactname}" | pv -batf -i 5 | unzstd | docker load + timeout 20m gh-actions-artifact-client.js download "${artifactname}" | pv -batf -i 5 | unzstd | docker load echo "::endgroup::" } @@ -134,7 +140,7 @@ function ci_restore_tar_from_github_actions_artifacts() { local artifactname="${1}.tar.zst" ci_install_tool pv echo "::group::Restoring tar from name ${artifactname} in GitHub Actions Artifacts to $PWD" - gh-actions-artifact-client.js download "${artifactname}" | pv -batf -i 5 | tar -I zstd -xf - + timeout 5m gh-actions-artifact-client.js download "${artifactname}" | pv -batf -i 5 | tar -I zstd -xf - echo "::endgroup::" } @@ -142,12 +148,18 @@ function ci_restore_tar_from_github_actions_artifacts() { function ci_store_tar_to_github_actions_artifacts() { local artifactname="${1}.tar.zst" shift - ci_install_tool pv - echo "::group::Storing $1 tar command output to name ${artifactname} in GitHub Actions Artifacts" - # delete possible previous artifact that might exist when re-running - gh-actions-artifact-client.js delete "${artifactname}" &>/dev/null || true - "$@" | pv -ft -i 5 | pv -Wbaf -i 5 | gh-actions-artifact-client.js upload --retentionDays=$ARTIFACT_RETENTION_DAYS "${artifactname}" - echo "::endgroup::" + if [[ "$GITHUB_ACTIONS" == "true" ]]; then + ci_install_tool pv + echo "::group::Storing $1 tar command output to name ${artifactname} in GitHub Actions Artifacts" + # delete possible previous artifact that might exist when re-running + timeout 1m gh-actions-artifact-client.js delete "${artifactname}" &>/dev/null || true + "$@" | pv -ft -i 5 | pv -Wbaf -i 5 | timeout 10m gh-actions-artifact-client.js upload --retentionDays=$ARTIFACT_RETENTION_DAYS "${artifactname}" + echo "::endgroup::" + else + local artifactfile="$(mktemp -t artifact.XXXX)" + echo "Storing output for debugging in $artifactfile" + "$@" | pv -ft -i 5 | pv -Wbaf -i 5 > $artifactfile + fi } # copies test reports into test-reports and surefire-reports directory @@ -284,6 +296,319 @@ EOF return 1 } +ci_snapshot_pulsar_maven_artifacts() { + ( + if [ -n "$GITHUB_WORKSPACE" ]; then + cd "$GITHUB_WORKSPACE" + else + fail "This script can only be run in GitHub Actions" + fi + mkdir -p target + find $HOME/.m2/repository/org/apache/pulsar -name "*.jar" > /tmp/provided_pulsar_maven_artifacts + ) +} + +ci_upload_unittest_coverage_files() { + _ci_upload_coverage_files unittest "$1" +} + +ci_upload_inttest_coverage_files() { + _ci_upload_coverage_files_inttest inttest "$1" integration-tests +} + +ci_upload_systest_coverage_files() { + _ci_upload_coverage_files_inttest systest "$1" system-tests +} + +_ci_upload_coverage_files_inttest() { + local testtype="$1" + local testgroup="$2" + local job_name="$3" + local store_deps=0 + local firsttestgroup="$(_ci_list_testgroups_with_coverage "${job_name}" | head -1)" + if [[ "${firsttestgroup}" == "${testgroup}" ]]; then + store_deps=1 + fi + _ci_upload_coverage_files "${testtype}" "${testgroup}" $store_deps +} + +_ci_upload_coverage_files() { + ( + testtype="$1" + testgroup="$2" + store_deps="${3:-1}" + echo "::group::Uploading $testtype coverage files" + if [ -n "$GITHUB_WORKSPACE" ]; then + cd "$GITHUB_WORKSPACE" + else + fail "This script can only be run in GitHub Actions" + fi + + if [ ! -f /tmp/provided_pulsar_maven_artifacts ]; then + fail "It is necessary to run '$0 snapshot_pulsar_maven_artifacts' before running any tests." + fi + + set -x + + local classpathFile="target/classpath_${testtype}_${testgroup}" + + local execFiles=$(find . -path "*/target/jacoco.exec" -printf "%P\n") + if [[ -n "$execFiles" ]]; then + # create temp file + local completeClasspathFile=$(mktemp -t tmp.classpath.XXXX) + + ci_install_tool xmlstarlet + + # iterate the exec files that were found + for execFile in $execFiles; do + local project="${execFile/%"/target/jacoco.exec"}" + local artifactId=$(xmlstarlet sel -t -m _:project -v _:artifactId -n $project/pom.xml) + local scope=runtime + # for integration tests, there's no dependencies in the runtime scope + # detect a plain test project based on missing src/main/java + if [ ! -d $project/src/main/java ]; then + scope=test + fi + # find the runtime classpath for the project to ensure that only production classes get covered + mvn -f $project/pom.xml -DincludeScope=$scope -Dscan=false dependency:build-classpath -B | { grep 'Dependencies classpath:' -A1 || true; } | tail -1 \ + | sed 's/:/\n/g' | { grep 'org/apache/pulsar' || true; } \ + | { tee -a $completeClasspathFile || true; } > target/classpath_$artifactId || true + done + + cat $completeClasspathFile | sort | uniq > $classpathFile + # delete temp file + rm $completeClasspathFile + + # upload target/jacoco.exec, target/classes and any dependent jar files that were built during the unit test execution + # transform jacoco exec filenames by appending "_${testtype}_${testgroup}" to the filename part to make the files unique + # so that they don't get overridden when all files are extracted to the same working directory before merging + ( + cd / + ci_store_tar_to_github_actions_artifacts coverage_and_deps_${testtype}_${testgroup} \ + tar -I zstd -cPf - \ + --transform="flags=r;s|/jacoco.exec$|/jacoco_${testtype}_${testgroup}.exec|" \ + --transform="flags=r;s|\\(/tmp/jacocoDir/.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \ + $GITHUB_WORKSPACE/target/classpath_* \ + $(find "$GITHUB_WORKSPACE" -path "*/target/jacoco.exec" -printf "%p\n%h/classes\n") \ + $([ -d /tmp/jacocoDir ] && echo "/tmp/jacocoDir" ) \ + $([[ $store_deps -eq 1 ]] && { cat $GITHUB_WORKSPACE/$classpathFile | sort | uniq | { grep -v -Fx -f /tmp/provided_pulsar_maven_artifacts || true; }; } || true) + ) + fi + echo "::endgroup::" + ) +} + +ci_restore_unittest_coverage_files() { + _ci_restore_coverage_files unittest unit-tests +} + +ci_restore_inttest_coverage_files() { + _ci_restore_coverage_files inttest integration-tests +} + +ci_restore_systest_coverage_files() { + _ci_restore_coverage_files systest system-tests +} + +_ci_restore_coverage_files() { + ( + test_type="$1" + job_name="$2" + cd / + for testgroup in $(_ci_list_testgroups_with_coverage "${job_name}"); do + ci_restore_tar_from_github_actions_artifacts coverage_and_deps_${test_type}_${testgroup} || true + done + ) +} + +_ci_list_testgroups_with_coverage() { + local job_name="$1" + yq e ".jobs.${job_name}.strategy.matrix.include.[] | select(.no_coverage != true) | .group" "$GITHUB_WORKSPACE/.github/workflows/pulsar-ci.yaml" +} + +ci_delete_unittest_coverage_files() { + _ci_delete_coverage_files unittest unit-tests +} + +ci_delete_inttest_coverage_files() { + _ci_delete_coverage_files inttest integration-tests +} + +ci_delete_systest_coverage_files() { + _ci_delete_coverage_files systest system-tests +} + +_ci_delete_coverage_files() { + ( + test_type="$1" + job_name="$2" + for testgroup in $(yq e ".jobs.${job_name}.strategy.matrix.include.[] | select(.no_coverage != true) | .group" "$GITHUB_WORKSPACE/.github/workflows/pulsar-ci.yaml"); do + timeout 1m gh-actions-artifact-client.js delete coverage_and_deps_${test_type}_${testgroup}.tar.zst || true + done + ) +} + +# creates an aggregated jacoco xml report for all projects that contain a target/jacoco.exec file +# +# the default maven jacoco report has multiple problems: +# - by default, jacoco:report goal will only report coverage for the current project. it is not suitable for Pulsar's +# unit tests that test production code that resides in multiple modules. +# - there's jacoco:report-aggregate that is supposed to resolve this. It has 2 issues: +# - 0.8.8 version doesn't yet support the required "includeCurrentProject" feature +# - the dependent projects must be built as part of the same mvn execution and belong to the same maven "reactor" +# - this isn't compatible with the way how Pulsar CI builds in "Build and License check" job and reuses +# the build results to run unit tests. +# +# This solution resolves the problem by using the Jacoco command line tool to generate the report. +# It assumes that all projects that contain a target/jacoco.exec file will also contain compiled classfiles. +ci_create_test_coverage_report() { + echo "::group::Create test coverage report" + if [ -n "$GITHUB_WORKSPACE" ]; then + cd "$GITHUB_WORKSPACE" + else + cd "$SCRIPT_DIR/.." + fi + local execFiles=$(find . '(' -path "*/target/jacoco.exec" -or -path "*/target/jacoco_*.exec" ')' -printf "%P\n") + if [[ -n "$execFiles" ]]; then + mkdir -p /tmp/jacocoDir + if [ ! -f /tmp/jacocoDir/jacococli.jar ]; then + local jacoco_version=$(mvn help:evaluate -Dscan=false -Dexpression=jacoco-maven-plugin.version -q -DforceStdout) + curl -sL -o /tmp/jacocoDir/jacococli.jar "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/${jacoco_version}/org.jacoco.cli-${jacoco_version}-nodeps.jar" + fi + + ci_install_tool xmlstarlet + # create mapping from project directory to project artifactId + local projectToArtifactIdMapping=$(find -name pom.xml -printf "%P\n" |xargs -I{} bash -c 'echo -n "$(dirname $1) "; xmlstarlet sel -t -m _:project -v _:artifactId -n $1' -- {}) + + # create temp files + local completeClasspathFile=$(mktemp -t tmp.classpath.XXXX) + local filterArtifactsFile=$(mktemp -t tmp.artifacts.XXXX) + local classesDir=$(mktemp -d -t tmp.classes.XXXX) + local sourcefilesFile=$(mktemp -t tmp.sources.XXXX) + + local projects=$({ + for execFile in $execFiles; do + echo $(dirname "$(dirname "$execFile")") + done + } | sort | uniq) + + # iterate projects + for project in $projects; do + local artifactId="$(printf "%s" "$projectToArtifactIdMapping" | grep -F "$project " | cut -d' ' -f2)" + if [ -d "$project/target/classes" ]; then + mkdir -p "$classesDir/$project" + cp -Rl "$project/target/classes" "$classesDir/$project" + echo "/$artifactId/" >> $filterArtifactsFile + fi + local scope=runtime + if [ -d $project/src/main/java ]; then + echo "$project/src/main/java" >> $sourcefilesFile + else + # for integration tests, there's no dependencies in the runtime scope + scope=test + fi + if [ -f "target/classpath_$artifactId" ]; then + echo "Found cached classpath for $artifactId." + cat "target/classpath_$artifactId" >> $completeClasspathFile + else + echo "Resolving classpath for $project..." + # find the runtime classpath for the project to ensure that only production classes get covered + mvn -f $project/pom.xml -DincludeScope=$scope -Dscan=false dependency:build-classpath -B | { grep 'Dependencies classpath:' -A1 || true; } | tail -1 \ + | sed 's/:/\n/g' | { grep 'org/apache/pulsar' || true; } \ + >> $completeClasspathFile || true + fi + done + + # delete any possible embedded jar files in the classes directory + find "$classesDir" -name "*.jar" -print -delete + + filterJarsPattern="bouncy-castle-bc|tests|/buildtools/" + + local classfilesArgs="--classfiles $({ + { + for classpathEntry in $(cat $completeClasspathFile | { grep -v -f $filterArtifactsFile || true; } | sort | uniq | { grep -v -E $filterJarsPattern || true; }); do + if [[ -f $classpathEntry && -n "$(unzip -Z1C $classpathEntry 'META-INF/bundled-dependencies/*' 2>/dev/null)" ]]; then + # file must be processed by removing META-INF/bundled-dependencies + local jartempfile=$(mktemp -t jarfile.XXXX --suffix=.jar) + cp $classpathEntry $jartempfile + zip -q -d $jartempfile 'META-INF/bundled-dependencies/*' &> /dev/null + echo $jartempfile + else + echo $classpathEntry + fi + done + } + echo $classesDir + } | tr '\n' ':' | sed -e 's/:$//' -e 's/:/ --classfiles /g')" + + local sourcefilesArgs="--sourcefiles $({ + # find the source file folders for the pulsar .jar files that are on the classpath + for artifactId in $(cat $completeClasspathFile | sort | uniq | { grep -v -E $filterJarsPattern || true; } | perl -p -e 's|.*/org/apache/pulsar/([^/]*)/.*|$1|'); do + local project="$(printf "%s" "$projectToArtifactIdMapping" | { grep $artifactId || true; } | cut -d' ' -f1)" + if [[ -n "$project" && -d "$project/src/main/java" ]]; then + echo "$project/src/main/java" + fi + done + cat $sourcefilesFile + } | tr '\n' ':' | sed -e 's/:$//' -e 's/:/ --sourcefiles /g')" + + rm $completeClasspathFile $filterArtifactsFile $sourcefilesFile + + set -x + mkdir -p target/jacoco_test_coverage_report/html + java -jar /tmp/jacocoDir/jacococli.jar report $execFiles \ + $classfilesArgs \ + --encoding UTF-8 --name "Apache Pulsar test coverage" \ + $sourcefilesArgs \ + --xml target/jacoco_test_coverage_report/jacoco.xml \ + --html target/jacoco_test_coverage_report/html \ + --csv target/jacoco_test_coverage_report/jacoco.csv + set +x + + rm -rf "$classesDir" + fi + echo "::endgroup::" +} + +# creates a jacoco xml report of the jacoco exec files produced in docker containers which have /tmp/jacocoDir mounted as /jacocoDir +# this is used to calculate test coverage for the apache/pulsar code that is run inside the containers in integration tests +# and system tests +ci_create_inttest_coverage_report() { + echo "::group::Create int test coverage in containers report" + if [[ -n "$(find /tmp/jacocoDir -name "*.exec" -print -quit)" ]]; then + cd "$GITHUB_WORKSPACE" + echo "Creating coverage report to target/jacoco_inttest_coverage_report" + set -x + mkdir -p target/jacoco_inttest_coverage_report + # install jacococli.jar command line tool + if [ ! -f /tmp/jacocoDir/jacococli.jar ]; then + local jacoco_version=$(mvn help:evaluate -Dexpression=jacoco-maven-plugin.version -Dscan=false -q -DforceStdout) + curl -sL -o /tmp/jacocoDir/jacococli.jar "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/${jacoco_version}/org.jacoco.cli-${jacoco_version}-nodeps.jar" + fi + # extract the Pulsar jar files from the docker image that was used to run the tests in the docker containers + # the class files used to produce the jacoco exec files are needed in the xml report generation + if [ ! -d /tmp/jacocoDir/pulsar_lib ]; then + mkdir /tmp/jacocoDir/pulsar_lib + docker run --rm -u "$UID:${GID:-"$(id -g)"}" -v /tmp/jacocoDir/pulsar_lib:/pulsar_lib:rw ${PULSAR_TEST_IMAGE_NAME:-apachepulsar/java-test-image:latest} bash -c "cp -p /pulsar/lib/org.apache.pulsar-* /pulsar_lib; [ -d /pulsar/connectors ] && cp -R /pulsar/connectors /pulsar_lib || true" + # remove jar file that causes duplicate classes issue + rm /tmp/jacocoDir/pulsar_lib/org.apache.pulsar-bouncy-castle* || true + # remove any bundled dependencies as part of .jar/.nar files + find /tmp/jacocoDir/pulsar_lib '(' -name "*.jar" -or -name "*.nar" ')' -exec echo "Processing {}" \; -exec zip -q -d {} 'META-INF/bundled-dependencies/*' \; |grep -E -v "Nothing to do|^$" || true + fi + # produce jacoco XML coverage report from the exec files and using the extracted jar files + java -jar /tmp/jacocoDir/jacococli.jar report /tmp/jacocoDir/*.exec \ + --classfiles /tmp/jacocoDir/pulsar_lib --encoding UTF-8 --name "Pulsar Integration Tests - coverage in containers" \ + $(find -path "*/src/main/java" -printf "--sourcefiles %P ") \ + --xml target/jacoco_inttest_coverage_report/jacoco.xml \ + --html target/jacoco_inttest_coverage_report/html \ + --csv target/jacoco_inttest_coverage_report/jacoco.csv + set +x + else + echo "No /tmp/jacocoDir/*.exec files to process" + fi + echo "::endgroup::" +} + if [ -z "$1" ]; then echo "usage: $0 [ci_tool_function_name]" echo "Available ci tool functions:" diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh index 974ad96e419ac..bc1255d8d68aa 100755 --- a/build/run_integration_group.sh +++ b/build/run_integration_group.sh @@ -27,14 +27,13 @@ set -o errexit JAVA_MAJOR_VERSION="$(java -version 2>&1 |grep " version " | awk -F\" '{ print $2 }' | awk -F. '{ if ($1=="1") { print $2 } else { print $1 } }')" # Used to shade run test on Java 8, because the latest TestNG requires Java 11 or higher. TESTNG_VERSION="7.3.0" -COVERAGE_COLLECTED=0 # lists all active maven modules with given parameters # parses the modules from the "mvn initialize" output # returns a CSV value mvn_list_modules() { ( - mvn -B -ntp "$@" initialize \ + mvn -B -ntp -Dscan=false "$@" initialize \ | grep -- "-< .* >-" \ | sed -E 's/.*-< (.*) >-.*/\1/' \ | tr '\n' ',' | sed 's/,$/\n/' @@ -85,7 +84,6 @@ mvn_run_integration_test() { sudo chmod 0777 /tmp/jacocoDir || chmod 0777 /tmp/jacocoDir fi coverage_args="-Pcoverage -Dintegrationtest.coverage.enabled=true -Dintegrationtest.coverage.dir=/tmp/jacocoDir" - COVERAGE_COLLECTED=1 shift fi ( @@ -109,38 +107,6 @@ mvn_run_integration_test() { ) } -# creates a jacoco xml report of the jacoco exec files produced in docker containers which have /tmp/jacocoDir mounted as /jacocoDir -# this is used to calculate test coverage for the apache/pulsar code that is run inside the containers in integration tests -# and system tests -produce_integration_test_coverage_xml_report() { - if [[ -d /tmp/jacocoDir && "$OSTYPE" == "linux-gnu"* && -n "$(find /tmp/jacocoDir -name "*.exec" -print -quit)" ]]; then - cd "$GITHUB_WORKSPACE" - local jacoco_xml_file=tests/integration/target/site/jacoco/jacoco_inttests.xml - echo "Creating coverage report to $jacoco_xml_file" - local jacoco_version=$(mvn help:evaluate -Dexpression=jacoco-maven-plugin.version -q -DforceStdout) - set -x - mkdir -p $(dirname $jacoco_xml_file) - # install jacococli.jar command line tool - if [ ! -f /tmp/jacocoDir/jacococli.jar ]; then - curl -sL -o /tmp/jacocoDir/jacococli.jar "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/${jacoco_version}/org.jacoco.cli-${jacoco_version}-nodeps.jar" - fi - # extract the Pulsar jar files from the docker image that was used to run the tests in the docker containers - # the class files used to produce the jacoco exec files are needed in the xml report generation - if [ ! -d /tmp/jacocoDir/pulsar_lib ]; then - mkdir /tmp/jacocoDir/pulsar_lib - docker run --rm -u "$UID:${GID:-"$(id -g)"}" -v /tmp/jacocoDir/pulsar_lib:/pulsar_lib:rw ${PULSAR_TEST_IMAGE_NAME:-apachepulsar/java-test-image:latest} bash -c "cp -p /pulsar/lib/org.apache.pulsar-* /pulsar_lib" - # remove jar file that causes duplicate classes issue - rm /tmp/jacocoDir/pulsar_lib/org.apache.pulsar-bouncy-castle* - fi - # produce jacoco XML coverage report from the exec files and using the extracted jar files - java -jar /tmp/jacocoDir/jacococli.jar report /tmp/jacocoDir/*.exec \ - --classfiles /tmp/jacocoDir/pulsar_lib --encoding UTF-8 --name "Pulsar Integration Tests - coverage in containers" \ - $(find -path "*/src/main/java" -printf "--sourcefiles %P ") \ - --xml $jacoco_xml_file - set +x - fi -} - test_group_shade() { mvn_run_integration_test "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false } @@ -266,9 +232,6 @@ echo "Test Group : $TEST_GROUP" test_group_function_name="test_group_$(echo "$TEST_GROUP" | tr '[:upper:]' '[:lower:]')" if [[ "$(LC_ALL=C type -t "${test_group_function_name}")" == "function" ]]; then eval "$test_group_function_name" "$@" - if [ $COVERAGE_COLLECTED -eq 1 ]; then - produce_integration_test_coverage_xml_report - fi else echo "INVALID TEST GROUP" echo "Available test groups:" diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 8546aff00668e..4d1e70ed3f91a 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -192,6 +192,11 @@ function list_test_groups() { # Test Groups -- end -- +if [[ "$1" == "--list" ]]; then + list_test_groups + exit 0 +fi + TEST_GROUP=$1 if [ -z "$TEST_GROUP" ]; then echo "usage: $0 [test_group]" diff --git a/codecov.yml b/codecov.yml index a792887012396..d8330c2ef3e81 100644 --- a/codecov.yml +++ b/codecov.yml @@ -20,18 +20,14 @@ codecov: require_ci_to_pass: yes notify: - # should match the number of builds sending coverage reports - # pulsar-ci.yaml contains: - # - 9 unit test builds - # - 4 integration test builds (without 'no_coverage: true') - # - 8 system test build - # pulsar-ci-flaky.yaml contains: - # - 1 build - after_n_builds: 22 + # should match the number of coverage report uploads + # pulsar-ci.yaml contains 3 uploads (unittests, inttests, systests) + # pulsar-ci-flaky.yaml contains 1 upload + after_n_builds: 4 comment: # should match the number of builds sending coverage reports - after_n_builds: 22 + after_n_builds: 4 layout: "reach, diff, flags, files" behavior: default require_changes: false diff --git a/pom.xml b/pom.xml index 0d323bfb1a881..f40e79dd48b85 100644 --- a/pom.xml +++ b/pom.xml @@ -113,6 +113,8 @@ flexible messaging model and an intuitive client API. false ${project.build.directory} + + true /tmp kill apachepulsar @@ -1959,14 +1961,20 @@ flexible messaging model and an intuitive client API. - pre-unit-test + prepare-agent prepare-agent + + + org.apache.pulsar.* + org.apache.bookkeeper.mledger.* + + - post-test - test + report + verify report @@ -1974,6 +1982,7 @@ flexible messaging model and an intuitive client API. META-INF/** + ${jacoco.report.skip} diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index d83ce2d891a5c..49901437622af 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -63,13 +63,13 @@ org.apache.pulsar - pulsar-client + pulsar-client-original ${project.version} test org.apache.pulsar - pulsar-client-admin + pulsar-client-admin-original ${project.version} test