From b290553334481f38b16bcd0494265cc1ea4dd217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 9 Sep 2022 09:03:48 +0200 Subject: [PATCH 1/4] [ci] Move owasp check into the main workflow --- .github/changes-filter.yaml | 6 ++ .github/workflows/ci-owasp-dep-check.yaml | 101 ---------------------- .github/workflows/pulsar-ci.yaml | 68 +++++++++++++++ 3 files changed, 74 insertions(+), 101 deletions(-) delete mode 100644 .github/workflows/ci-owasp-dep-check.yaml diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml index b14153acb8242..b836dbf3091f1 100644 --- a/.github/changes-filter.yaml +++ b/.github/changes-filter.yaml @@ -12,3 +12,9 @@ tests: - added|modified: '**/src/test/java/**/*.java' cpp: - 'pulsar-client-cpp/**' +need_owasp: + - 'pom.xml' + - '**/pom.xml' + - 'src/owasp-dependency-check-false-positives.xml' + - 'src/owasp-dependency-check-suppressions.xml' + - '.github/workflows/ci-owasp-dep-check.yaml' \ No newline at end of file diff --git a/.github/workflows/ci-owasp-dep-check.yaml b/.github/workflows/ci-owasp-dep-check.yaml deleted file mode 100644 index 27f820fb20a0b..0000000000000 --- a/.github/workflows/ci-owasp-dep-check.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Misc - OWASP Dependency Check -on: - pull_request: - branches: - - master - push: - branches: - - branch-* - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - - owasp-dep-check: - name: - runs-on: ubuntu-20.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed pom files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: | - poms: - - 'pom.xml' - - '**/pom.xml' - - 'src/owasp-dependency-check-false-positives.xml' - - 'src/owasp-dependency-check-suppressions.xml' - - '.github/workflows/ci-owasp-dep-check.yaml' - - - name: Cache local Maven repository - if: ${{ steps.changes.outputs.poms == 'true' }} - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v2 - if: ${{ steps.changes.outputs.poms == 'true' }} - with: - distribution: 'temurin' - java-version: 17 - - - name: clean disk - if: ${{ steps.changes.outputs.poms == 'true' }} - run: | - sudo swapoff -a - sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - # Projects dependent on flume, hdfs, hbase, and trino currently excluded from the scan. - - name: run "clean verify" to trigger dependency check - if: ${{ steps.changes.outputs.poms == 'true' }} - run: mvn -q -B -ntp clean verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' - - - name: Upload report - uses: actions/upload-artifact@v3 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: dependency report - path: target/dependency-check-report.html diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 2f2e63ab36751..5532aea0f8406 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -855,6 +855,74 @@ jobs: - name: build package run: mvn -B clean package -DskipTests -T 1C -ntp + owasp-dep-check: + name: OWASP dependency check + runs-on: ubuntu-20.04 + timeout-minutes: 120 + needs: [ 'changed_files_job', 'integration-tests' ] + if: ${{ needs.changed_files_job.outputs.need_owasp == 'true' }} + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' }} + uses: ./.github/actions/ssh-access + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v2 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + - name: Set up JDK ${{ matrix.jdk || '17' }} + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk || '17' }} + + - name: clean disk + run: | + sudo swapoff -a + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. + - name: run "clean verify" to trigger dependency check + run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' + + - name: Upload report + uses: actions/upload-artifact@v3 + if: ${{ cancelled() || failure() }} + continue-on-error: true + with: + name: dependency report + path: target/dependency-check-report.html + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' }} + continue-on-error: true + with: + action: wait # This job is required for pulls to be merged. # It depends on all other jobs in this workflow. From 04503f6099b4b5e46179948f7a09689ffb2bc498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 9 Sep 2022 09:28:39 +0200 Subject: [PATCH 2/4] move after itests --- .github/workflows/pulsar-ci.yaml | 74 ++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 5532aea0f8406..ab558552c4a84 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -466,6 +466,80 @@ jobs: run: | gh-actions-artifact-client.js delete pulsar-java-test-image.zst + owasp-dep-check: + name: OWASP dependency check + runs-on: ubuntu-20.04 + timeout-minutes: 120 + # Put it after integration-tests to run only if the branch is stable + needs: [ 'integration-tests' ] + if: ${{ needs.changed_files_job.outputs.need_owasp == 'true' }} + + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' }} + uses: ./.github/actions/ssh-access + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v2 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Set up JDK 17 + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: 17 + + - name: clean disk + run: | + sudo swapoff -a + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. + - name: run "verify" to trigger dependency check + run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' + + - name: Upload report + uses: actions/upload-artifact@v3 + if: ${{ cancelled() || failure() }} + continue-on-error: true + with: + name: dependency report + path: target/dependency-check-report.html + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' }} + continue-on-error: true + with: + action: wait + pulsar-test-latest-version-image: name: Build Pulsar docker image runs-on: ubuntu-20.04 From 2e7db7d2ce815f6926d945382475c52176386067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 12 Sep 2022 13:15:38 +0200 Subject: [PATCH 3/4] fix --- .github/workflows/pulsar-ci.yaml | 74 -------------------------------- 1 file changed, 74 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index ab558552c4a84..5532aea0f8406 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -466,80 +466,6 @@ jobs: run: | gh-actions-artifact-client.js delete pulsar-java-test-image.zst - owasp-dep-check: - name: OWASP dependency check - runs-on: ubuntu-20.04 - timeout-minutes: 120 - # Put it after integration-tests to run only if the branch is stable - needs: [ 'integration-tests' ] - if: ${{ needs.changed_files_job.outputs.need_owasp == 'true' }} - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Setup ssh access to build runner VM - # ssh access is enabled for builds in own forks - if: ${{ github.repository != 'apache/pulsar' }} - uses: ./.github/actions/ssh-access - with: - limit-access-to-actor: true - - - name: Cache Maven dependencies - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 17 - - - name: clean disk - run: | - sudo swapoff -a - sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: Install gh-actions-artifact-client.js - uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master - - - name: Restore maven build results from Github artifact cache - run: | - cd $HOME - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - - # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. - - name: run "verify" to trigger dependency check - run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' - - - name: Upload report - uses: actions/upload-artifact@v3 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: dependency report - path: target/dependency-check-report.html - - - name: Wait for ssh connection when build fails - # ssh access is enabled for builds in own forks - uses: ./.github/actions/ssh-access - if: ${{ failure() && github.repository != 'apache/pulsar' }} - continue-on-error: true - with: - action: wait - pulsar-test-latest-version-image: name: Build Pulsar docker image runs-on: ubuntu-20.04 From 7759d9d6419230b8e6e93d9b25d59d15e7231235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 12 Sep 2022 13:25:47 +0200 Subject: [PATCH 4/4] fix filter --- .github/changes-filter.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml index b836dbf3091f1..82f5b22a8aad9 100644 --- a/.github/changes-filter.yaml +++ b/.github/changes-filter.yaml @@ -16,5 +16,4 @@ need_owasp: - 'pom.xml' - '**/pom.xml' - 'src/owasp-dependency-check-false-positives.xml' - - 'src/owasp-dependency-check-suppressions.xml' - - '.github/workflows/ci-owasp-dep-check.yaml' \ No newline at end of file + - 'src/owasp-dependency-check-suppressions.xml' \ No newline at end of file