diff --git a/.dryrunsecurity.yaml b/.dryrunsecurity.yaml index 740ae9ba1bc..1863e9a2027 100644 --- a/.dryrunsecurity.yaml +++ b/.dryrunsecurity.yaml @@ -52,7 +52,6 @@ sensitiveCodepaths: - 'docker/entrypoint-celery-beat.sh' - 'docker/entrypoint-celery-worker.sh' - 'docker/entrypoint-initializer.sh' - - 'docker/entrypoint-first-boot.sh' - 'docker/entrypoint-nginx.sh' - 'docker/entrypoint-uwsgi.sh' - 'docker/wait-for-it.sh' diff --git a/.github/workflows/build-docker-images-for-testing.yml b/.github/workflows/build-docker-images-for-testing.yml index 052fb5896a7..4d1f1147f84 100644 --- a/.github/workflows/build-docker-images-for-testing.yml +++ b/.github/workflows/build-docker-images-for-testing.yml @@ -40,7 +40,7 @@ jobs: echo $GITHUB_ENV - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false @@ -49,7 +49,7 @@ jobs: run: echo "IMAGE_REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build id: docker_build @@ -67,7 +67,7 @@ jobs: # export docker images to be used in next jobs below - name: Upload image ${{ matrix.docker-image }} as artifact timeout-minutes: 15 - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: built-docker-image-${{ matrix.docker-image }}-${{ matrix.os }}-${{ env.PLATFORM }} path: ${{ matrix.docker-image }}-${{ matrix.os }}-${{ env.PLATFORM }}_img diff --git a/.github/workflows/close-stale.yml b/.github/workflows/close-stale.yml index 857f619c78b..9651b0c341c 100644 --- a/.github/workflows/close-stale.yml +++ b/.github/workflows/close-stale.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Close issues and PRs that are pending closure - uses: actions/stale@5f858e3efba33a5ca4407a664cc011ad407f2008 # v10.1.0 + uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v10.1.1 with: # Disable automatic stale marking - only close manually labeled items days-before-stale: -1 @@ -27,7 +27,7 @@ jobs: close-pr-message: 'This PR has been automatically closed because it was manually labeled as stale. If you believe this was closed in error, please reopen it and remove the stale label.' - name: Close stale issues and PRs - uses: actions/stale@5f858e3efba33a5ca4407a664cc011ad407f2008 # v10.1.0 + uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v10.1.1 with: # Disable automatic stale marking - only close manually labeled items days-before-stale: -1 diff --git a/.github/workflows/fetch-oas.yml b/.github/workflows/fetch-oas.yml index e36da098b4c..35143d98b28 100644 --- a/.github/workflows/fetch-oas.yml +++ b/.github/workflows/fetch-oas.yml @@ -22,7 +22,7 @@ jobs: file-type: [yaml, json] steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: release/${{ env.release_version }} @@ -55,7 +55,7 @@ jobs: run: docker compose down - name: Upload oas.${{ matrix.file-type }} as artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: oas-${{ matrix.file-type }} path: oas.${{ matrix.file-type }} diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 2c383433fd7..e033e2da335 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -18,16 +18,16 @@ jobs: - name: Setup Hugo uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0 with: - hugo-version: '0.152.2' # renovate: datasource=github-releases depName=gohugoio/hugo + hugo-version: '0.153.4' # renovate: datasource=github-releases depName=gohugoio/hugo extended: true - name: Setup Node - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 with: - node-version: '24.11.1' # TODO: Renovate helper might not be needed here - needs to be fully tested + node-version: '24.12.0' # TODO: Renovate helper might not be needed here - needs to be fully tested - name: Cache dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} @@ -35,7 +35,7 @@ jobs: ${{ runner.os }}-node- - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: submodules: recursive fetch-depth: 0 @@ -46,13 +46,12 @@ jobs: - name: Install dependencies run: cd docs && npm ci - + - name: Build production website env: HUGO_ENVIRONMENT: production HUGO_ENV: production run: cd docs && hugo --minify --gc --config config/production/hugo.toml - - name: Deploy uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 if: github.repository == 'DefectDojo/django-DefectDojo' # Deploy docs only in core repo, not in forks - it would just fail in fork diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d1f1bbab941..5ed6e25a993 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -2,18 +2,12 @@ name: Integration tests on: workflow_call: - inputs: - auditlog_type: - type: string - default: "django-auditlog" jobs: integration_tests: # run tests with docker compose name: User Interface Tests runs-on: ubuntu-latest - env: - AUDITLOG_TYPE: ${{ inputs.auditlog_type }} strategy: matrix: test-case: [ @@ -54,11 +48,11 @@ jobs: steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 # load docker images from build jobs - name: Load images from artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: built-docker-image pattern: built-docker-image-* diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index a96dbfa7bee..ad67880be21 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -16,13 +16,13 @@ jobs: # databases, broker and k8s are independent, so we don't need to test each combination # lastest k8s version (https://kubernetes.io/releases/) and the oldest officially supported version # are tested (https://kubernetes.io/releases/) - - k8s: 'v1.34.2' # renovate: datasource=github-releases depName=kubernetes/kubernetes versioning=loose + - k8s: 'v1.35.0' # renovate: datasource=github-releases depName=kubernetes/kubernetes versioning=loose os: debian - - k8s: '1.32.10' # renovate: datasource=custom.endoflife-oldest-maintained depName=kubernetes + - k8s: '1.32.11' # renovate: datasource=custom.endoflife-oldest-maintained depName=kubernetes os: debian steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Setup Minikube uses: manusa/actions-setup-minikube@b589f2d61bf96695c546929c72b38563e856059d # v2.14.0 @@ -38,7 +38,7 @@ jobs: minikube status - name: Load images from artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: built-docker-image pattern: built-docker-image-* diff --git a/.github/workflows/release-1-create-pr.yml b/.github/workflows/release-1-create-pr.yml index 2c0cd53c786..39d7d4453e4 100644 --- a/.github/workflows/release-1-create-pr.yml +++ b/.github/workflows/release-1-create-pr.yml @@ -40,7 +40,7 @@ jobs: run: echo "GITHUB_ORG=${GITHUB_REPOSITORY%%/*}" >> $GITHUB_ENV - name: Checkout from_branch branch - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: ${{ inputs.from_branch }} @@ -58,7 +58,7 @@ jobs: run: git push origin HEAD:${NEW_BRANCH} - name: Checkout release branch - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: ${{ env.NEW_BRANCH }} @@ -98,7 +98,7 @@ jobs: chart-search-root: "helm/defectdojo" - name: Push version changes - uses: stefanzweifel/git-auto-commit-action@28e16e81777b558cc906c8750092100bbb34c5e3 # v7.0.0 + uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # v7.1.0 with: commit_user_name: "${{ env.GIT_USERNAME }}" commit_user_email: "${{ env.GIT_EMAIL }}" diff --git a/.github/workflows/release-2-tag-docker-push.yml b/.github/workflows/release-2-tag-docker-push.yml index dd3369eadd1..112e8ccecbd 100644 --- a/.github/workflows/release-2-tag-docker-push.yml +++ b/.github/workflows/release-2-tag-docker-push.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: master diff --git a/.github/workflows/release-3-master-into-dev.yml b/.github/workflows/release-3-master-into-dev.yml index 708b9f31c44..ce757d875a4 100644 --- a/.github/workflows/release-3-master-into-dev.yml +++ b/.github/workflows/release-3-master-into-dev.yml @@ -23,7 +23,7 @@ jobs: run: echo "GITHUB_ORG=${GITHUB_REPOSITORY%%/*}" >> $GITHUB_ENV - name: Checkout master - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: master @@ -40,7 +40,7 @@ jobs: run: git push origin HEAD:${NEW_BRANCH} - name: Checkout new branch - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: ${{ env.NEW_BRANCH }} @@ -86,7 +86,7 @@ jobs: chart-search-root: "helm/defectdojo" - name: Push version changes - uses: stefanzweifel/git-auto-commit-action@28e16e81777b558cc906c8750092100bbb34c5e3 # v7.0.0 + uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # v7.1.0 with: commit_user_name: "${{ env.GIT_USERNAME }}" commit_user_email: "${{ env.GIT_EMAIL }}" @@ -115,7 +115,7 @@ jobs: run: echo "GITHUB_ORG=${GITHUB_REPOSITORY%%/*}" >> $GITHUB_ENV - name: Checkout master - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: master @@ -132,7 +132,7 @@ jobs: run: git push origin HEAD:${NEW_BRANCH} - name: Checkout new branch - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: ${{ env.NEW_BRANCH }} @@ -162,7 +162,7 @@ jobs: chart-search-root: "helm/defectdojo" - name: Push version changes - uses: stefanzweifel/git-auto-commit-action@28e16e81777b558cc906c8750092100bbb34c5e3 # v7.0.0 + uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # v7.1.0 with: commit_user_name: "${{ env.GIT_USERNAME }}" commit_user_email: "${{ env.GIT_EMAIL }}" diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index baa804441a0..5f1b5968be3 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -47,7 +47,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Load OAS files from artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: pattern: oas-* diff --git a/.github/workflows/release-x-manual-docker-containers.yml b/.github/workflows/release-x-manual-docker-containers.yml index b376923d5b4..8e73c08b6e7 100644 --- a/.github/workflows/release-x-manual-docker-containers.yml +++ b/.github/workflows/release-x-manual-docker-containers.yml @@ -58,13 +58,13 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Checkout tag - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: ${{ inputs.release_number }} - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 # we cannot set any tags here, those are set on the merged digest in release-x-manual-merge-container-digests.yml - name: Build and push images @@ -76,6 +76,7 @@ jobs: push: true file: ./Dockerfile.${{ matrix.docker-image }}-${{ matrix.os }} context: . + target: release outputs: type=image,"name=${{ env.DOCKER_ORG }}/defectdojo-${{ matrix.docker-image}}",push-by-digest=true,name-canonical=true cache-from: type=gha,scope=${{ matrix.docker-image}}-${{ matrix.os }}-${{ env.PLATFORM }}-${{ github.head_ref || github.ref_name }} cache-to: type=gha,mode=max,scope=${{ matrix.docker-image}}-${{ matrix.os }}-${{ env.PLATFORM }}-${{ github.head_ref || github.ref_name }} @@ -89,7 +90,7 @@ jobs: # upload the digest file as artifact - name: Upload digest - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: digests-${{ matrix.docker-image}}-${{ matrix.os }}-${{ env.PLATFORM }} path: ${{ runner.temp }}/digests/* diff --git a/.github/workflows/release-x-manual-helm-chart.yml b/.github/workflows/release-x-manual-helm-chart.yml index 719071c68e0..0ee1d731bbf 100644 --- a/.github/workflows/release-x-manual-helm-chart.yml +++ b/.github/workflows/release-x-manual-helm-chart.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: ${{ inputs.release_number }} fetch-depth: 0 @@ -77,7 +77,7 @@ jobs: echo "chart_version=$(ls build | cut -d '-' -f 2,3 | sed 's|\.tgz||')" >> $GITHUB_ENV - name: Create release ${{ inputs.release_number }} - uses: softprops/action-gh-release@5be0e66d93ac7ed76da52eca8bb058f665c3a5fe # v2.4.2 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 with: name: '${{ inputs.release_number }} 🌈' tag_name: ${{ inputs.release_number }} diff --git a/.github/workflows/release-x-manual-merge-container-digests.yml b/.github/workflows/release-x-manual-merge-container-digests.yml index 156d3dfb28f..27cafd21731 100644 --- a/.github/workflows/release-x-manual-merge-container-digests.yml +++ b/.github/workflows/release-x-manual-merge-container-digests.yml @@ -41,7 +41,7 @@ jobs: # only download digests for this image and this os - name: Download digests - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: ${{ runner.temp }}/digests pattern: digests-${{ matrix.docker-image}}-${{ matrix.os }}-* @@ -54,7 +54,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 # the alpine and debian images are tagged with the os name - name: Create OS specific manifest list and push diff --git a/.github/workflows/release-x-manual-tag-as-latest.yml b/.github/workflows/release-x-manual-tag-as-latest.yml index 0a3d447edd1..2424dc436a2 100644 --- a/.github/workflows/release-x-manual-tag-as-latest.yml +++ b/.github/workflows/release-x-manual-tag-as-latest.yml @@ -43,7 +43,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Tag with latest tags run: | diff --git a/.github/workflows/release-x-nightly.yml b/.github/workflows/release-x-nightly.yml index 9ce48ef4254..d10f1d87050 100644 --- a/.github/workflows/release-x-nightly.yml +++ b/.github/workflows/release-x-nightly.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: ${{ inputs.branch-to-build }} diff --git a/.github/workflows/renovate.yaml b/.github/workflows/renovate.yaml index fdb4ae1b5fd..08206d44b17 100644 --- a/.github/workflows/renovate.yaml +++ b/.github/workflows/renovate.yaml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false @@ -21,4 +21,4 @@ jobs: uses: suzuki-shunsuke/github-action-renovate-config-validator@c22827f47f4f4a5364bdba19e1fe36907ef1318e # v1.1.1 with: strict: "true" - validator_version: 42.27.0 # renovate: datasource=github-releases depName=renovatebot/renovate + validator_version: 42.71.0 # renovate: datasource=github-releases depName=renovatebot/renovate diff --git a/.github/workflows/rest-framework-tests.yml b/.github/workflows/rest-framework-tests.yml index 7f857b4c05c..9dac672a9f6 100644 --- a/.github/workflows/rest-framework-tests.yml +++ b/.github/workflows/rest-framework-tests.yml @@ -6,16 +6,11 @@ on: platform: type: string default: "linux/amd64" - auditlog_type: - type: string - default: "django-auditlog" jobs: unit_tests: name: Rest Framework Unit Tests runs-on: ${{ inputs.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - env: - AUDITLOG_TYPE: ${{ inputs.auditlog_type }} strategy: matrix: @@ -30,13 +25,13 @@ jobs: echo $GITHUB_ENV - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false # load docker images from build jobs - name: Load images from artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: built-docker-image pattern: built-docker-image-* diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 58a56dc5aa0..3eb100c8b58 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install Ruff Linter run: pip install -r requirements-lint.txt diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index cb53b4b76ad..e42f9a46e2d 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Run ShellCheck uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # 2.0.0 diff --git a/.github/workflows/test-helm-chart.yml b/.github/workflows/test-helm-chart.yml index b54fd5bf5ba..2de8dc737cd 100644 --- a/.github/workflows/test-helm-chart.yml +++ b/.github/workflows/test-helm-chart.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false fetch-depth: 0 @@ -111,7 +111,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.event.pull_request.head.ref }} @@ -152,10 +152,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Generate values schema json - uses: losisin/helm-values-schema-json-action@660c441a4a507436a294fc55227e1df54aca5407 # v2.3.1 + uses: losisin/helm-values-schema-json-action@f3517c55537e26953c8a11be7549ea908990130d # v2.3.2 with: fail-on-diff: true working-directory: "helm/defectdojo" @@ -172,7 +172,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false fetch-depth: 0 @@ -194,7 +194,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Run ah lint working-directory: ./helm/defectdojo run: |- diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index cbda2b40caf..e16990520df 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -25,26 +25,18 @@ jobs: strategy: matrix: platform: ['linux/amd64', 'linux/arm64'] - auditlog_type: ['django-auditlog', 'django-pghistory'] fail-fast: false needs: build-docker-containers uses: ./.github/workflows/rest-framework-tests.yml secrets: inherit with: platform: ${{ matrix.platform}} - auditlog_type: ${{ matrix.auditlog_type }} # only run integration tests for linux/amd64 (default) test-user-interface: needs: build-docker-containers uses: ./.github/workflows/integration-tests.yml secrets: inherit - strategy: - matrix: - auditlog_type: ['django-auditlog', 'django-pghistory'] - fail-fast: false - with: - auditlog_type: ${{ matrix.auditlog_type }} # only run k8s tests for linux/amd64 (default) test-k8s: diff --git a/.github/workflows/update-sample-data.yml b/.github/workflows/update-sample-data.yml index e208e57a46a..6c5fd95ec4f 100644 --- a/.github/workflows/update-sample-data.yml +++ b/.github/workflows/update-sample-data.yml @@ -16,7 +16,7 @@ jobs: steps: # Checkout the repository - name: Checkout code - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: ref: ${{ github.ref_name || 'dev'}} @@ -43,7 +43,7 @@ jobs: git push --set-upstream origin $(git rev-parse --abbrev-ref HEAD) - name: Create Pull Request - uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: "Update sample data" diff --git a/.github/workflows/validate_docs_build.yml b/.github/workflows/validate_docs_build.yml index f83d6d189b8..6645999a10f 100644 --- a/.github/workflows/validate_docs_build.yml +++ b/.github/workflows/validate_docs_build.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - 'docs/**' + - '.github/workflows/*' jobs: deploy: @@ -12,16 +13,16 @@ jobs: - name: Setup Hugo uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0 with: - hugo-version: '0.152.2' # renovate: datasource=github-releases depName=gohugoio/hugo + hugo-version: '0.153.4' # renovate: datasource=github-releases depName=gohugoio/hugo extended: true - name: Setup Node - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 with: - node-version: '24.11.1' # TODO: Renovate helper might not be needed here - needs to be fully tested + node-version: '24.12.0' # TODO: Renovate helper might not be needed here - needs to be fully tested - name: Cache dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} @@ -29,7 +30,7 @@ jobs: ${{ runner.os }}-node- - name: Checkout - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: submodules: recursive fetch-depth: 0 diff --git a/Dockerfile.django-alpine b/Dockerfile.django-alpine index 40365930275..cd24ca38d4b 100644 --- a/Dockerfile.django-alpine +++ b/Dockerfile.django-alpine @@ -5,7 +5,7 @@ # Dockerfile.nginx to use the caching mechanism of Docker. # Ref: https://devguide.python.org/#branchstatus -FROM python:3.13.7-alpine3.22@sha256:9ba6d8cbebf0fb6546ae71f2a1c14f6ffd2fdab83af7fa5669734ef30ad48844 AS base +FROM python:3.13.11-alpine3.22@sha256:ab45bd32143151fe060d48218b91df43a289166e72ec7877823b1c972580bed3 AS base FROM base AS build WORKDIR /app RUN \ @@ -68,26 +68,20 @@ RUN \ COPY \ docker/entrypoint-celery-beat.sh \ docker/entrypoint-celery-worker.sh \ - docker/entrypoint-celery-worker-dev.sh \ docker/entrypoint-initializer.sh \ - docker/entrypoint-first-boot.sh \ docker/entrypoint-uwsgi.sh \ - docker/entrypoint-uwsgi-dev.sh \ - docker/entrypoint-unit-tests.sh \ - docker/entrypoint-unit-tests-devDocker.sh \ docker/wait-for-it.sh \ docker/secret-file-loader.sh \ docker/reach_database.sh \ docker/reach_broker.sh \ docker/certs/* \ / -COPY wsgi.py manage.py docker/unit-tests.sh ./ +COPY wsgi.py manage.py ./ COPY dojo/ ./dojo/ # Add extra fixtures to docker image which are loaded by the initializer COPY docker/extra_fixtures/* /app/dojo/fixtures/ -COPY tests/ ./tests/ RUN \ # Remove placeholder copied from docker/certs rm -f /readme.txt && \ @@ -140,9 +134,20 @@ ENTRYPOINT ["/entrypoint-uwsgi.sh"] FROM release AS development USER root -COPY requirements-dev.txt ./ -RUN pip3 install --no-cache-dir -r requirements-dev.txt +COPY \ + requirements-dev.txt \ + docker/unit-tests.sh \ + ./ +COPY \ + docker/entrypoint-celery-worker-dev.sh \ + docker/entrypoint-uwsgi-dev.sh \ + docker/entrypoint-unit-tests.sh \ + docker/entrypoint-unit-tests-devDocker.sh \ + / +RUN pip3 install --no-cache-dir -r requirements-dev.txt && \ + chmod 775 /*.sh USER ${uid} FROM development AS django-unittests COPY unittests/ ./unittests/ +COPY tests/ ./tests/ diff --git a/Dockerfile.django-debian b/Dockerfile.django-debian index eccf9bd6dae..12e0b48b0fc 100644 --- a/Dockerfile.django-debian +++ b/Dockerfile.django-debian @@ -5,7 +5,7 @@ # Dockerfile.nginx to use the caching mechanism of Docker. # Ref: https://devguide.python.org/#branchstatus -FROM python:3.13.7-slim-trixie@sha256:5f55cdf0c5d9dc1a415637a5ccc4a9e18663ad203673173b8cda8f8dcacef689 AS base +FROM python:3.13.11-slim-trixie@sha256:1f3781f578e17958f55ada96c0a827bf279a11e10d6a458ecb8bde667afbb669 AS base FROM base AS build WORKDIR /app RUN \ @@ -71,26 +71,20 @@ RUN \ COPY \ docker/entrypoint-celery-beat.sh \ docker/entrypoint-celery-worker.sh \ - docker/entrypoint-celery-worker-dev.sh \ docker/entrypoint-initializer.sh \ - docker/entrypoint-first-boot.sh \ docker/entrypoint-uwsgi.sh \ - docker/entrypoint-uwsgi-dev.sh \ - docker/entrypoint-unit-tests.sh \ - docker/entrypoint-unit-tests-devDocker.sh \ docker/wait-for-it.sh \ docker/secret-file-loader.sh \ docker/reach_database.sh \ docker/reach_broker.sh \ docker/certs/* \ / -COPY wsgi.py manage.py docker/unit-tests.sh ./ +COPY wsgi.py manage.py ./ COPY dojo/ ./dojo/ # Add extra fixtures to docker image which are loaded by the initializer COPY docker/extra_fixtures/* /app/dojo/fixtures/ -COPY tests/ ./tests/ RUN \ # Remove placeholder copied from docker/certs rm -f /readme.txt && \ @@ -143,9 +137,20 @@ ENTRYPOINT ["/entrypoint-uwsgi.sh"] FROM release AS development USER root -COPY requirements-dev.txt ./ -RUN pip3 install --no-cache-dir -r requirements-dev.txt +COPY \ + requirements-dev.txt \ + docker/unit-tests.sh \ + ./ +COPY \ + docker/entrypoint-celery-worker-dev.sh \ + docker/entrypoint-uwsgi-dev.sh \ + docker/entrypoint-unit-tests.sh \ + docker/entrypoint-unit-tests-devDocker.sh \ + / +RUN pip3 install --no-cache-dir -r requirements-dev.txt && \ + chmod 775 /*.sh USER ${uid} FROM development AS django-unittests COPY unittests/ ./unittests/ +COPY tests/ ./tests/ diff --git a/Dockerfile.integration-tests-debian b/Dockerfile.integration-tests-debian index 0b7c1d75b1c..4bd2caea2aa 100644 --- a/Dockerfile.integration-tests-debian +++ b/Dockerfile.integration-tests-debian @@ -1,9 +1,9 @@ # code: language=Dockerfile -FROM openapitools/openapi-generator-cli:v7.17.0@sha256:868b97eb4e5080d2cdfd5b3eeaa4d52e4bbb7c56f14e234b08b0b0bc4f38a78f AS openapitools +FROM openapitools/openapi-generator-cli:v7.18.0@sha256:be5c0a17c978ed4c39985312af3129882407581e07f2e3167cf777c908ffd52b AS openapitools # currently only supports x64, no arm yet due to chrome and selenium dependencies -FROM python:3.13.7-slim-trixie@sha256:5f55cdf0c5d9dc1a415637a5ccc4a9e18663ad203673173b8cda8f8dcacef689 AS build +FROM python:3.13.11-slim-trixie@sha256:1f3781f578e17958f55ada96c0a827bf279a11e10d6a458ecb8bde667afbb669 AS build WORKDIR /app RUN \ apt-get -y update && \ diff --git a/Dockerfile.nginx-alpine b/Dockerfile.nginx-alpine index aa867828a2f..03a6e9c5a31 100644 --- a/Dockerfile.nginx-alpine +++ b/Dockerfile.nginx-alpine @@ -5,7 +5,7 @@ # Dockerfile.django-alpine to use the caching mechanism of Docker. # Ref: https://devguide.python.org/#branchstatus -FROM python:3.13.7-alpine3.22@sha256:9ba6d8cbebf0fb6546ae71f2a1c14f6ffd2fdab83af7fa5669734ef30ad48844 AS base +FROM python:3.13.11-alpine3.22@sha256:ab45bd32143151fe060d48218b91df43a289166e72ec7877823b1c972580bed3 AS base FROM base AS build WORKDIR /app RUN \ @@ -63,7 +63,7 @@ COPY dojo/ ./dojo/ # always collect static for debug toolbar as we can't make it dependant on env variables or build arguments without breaking docker layer caching RUN env DD_SECRET_KEY='.' DD_DJANGO_DEBUG_TOOLBAR_ENABLED=True python3 manage.py collectstatic --noinput --verbosity=2 && true -FROM nginx:1.29.3-alpine3.22@sha256:b3c656d55d7ad751196f21b7fd2e8d4da9cb430e32f646adcf92441b72f82b14 +FROM nginx:1.29.3-alpine3.22@sha256:b3c656d55d7ad751196f21b7fd2e8d4da9cb430e32f646adcf92441b72f82b14 AS release ARG uid=1001 ARG appuser=defectdojo COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/ diff --git a/components/node_modules/.gitkeep b/components/node_modules/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/components/package.json b/components/package.json index 0057e739952..348ad4248bc 100644 --- a/components/package.json +++ b/components/package.json @@ -1,6 +1,6 @@ { "name": "defectdojo", - "version": "2.53.5", + "version": "2.54.0", "license" : "BSD-3-Clause", "private": true, "dependencies": { @@ -12,8 +12,8 @@ "chosen-bootstrap": "https://github.com/dbtek/chosen-bootstrap", "chosen-js": "^1.8.7", "clipboard": "^2.0.11", - "datatables.net": "^2.3.5", - "datatables.net-buttons-bs": "^3.2.5", + "datatables.net": "^2.3.6", + "datatables.net-buttons-bs": "^3.2.6", "datatables.net-colreorder": "^2.1.2", "drmonty-datatables-plugins": "^1.0.0", "drmonty-datatables-responsive": "^1.0.0", @@ -33,7 +33,7 @@ "metismenu": "~3.0.7", "moment": "^2.30.1", "morris.js": "morrisjs/morris.js", - "pdfmake": "^0.2.20", + "pdfmake": "^0.3.0", "startbootstrap-sb-admin-2": "1.0.7" }, "engines": { diff --git a/components/yarn.lock b/components/yarn.lock index 6c1c95ef183..8f2f00bbfa1 100644 --- a/components/yarn.lock +++ b/components/yarn.lock @@ -2,43 +2,12 @@ # yarn lockfile v1 -"@foliojs-fork/fontkit@^1.9.2": - version "1.9.2" - resolved "https://registry.yarnpkg.com/@foliojs-fork/fontkit/-/fontkit-1.9.2.tgz#94241c195bc6204157bc84c33f34bdc967eca9c3" - integrity sha512-IfB5EiIb+GZk+77TRB86AHroVaqfq8JRFlUbz0WEwsInyCG0epX2tCPOy+UfaWPju30DeVoUAXfzWXmhn753KA== - dependencies: - "@foliojs-fork/restructure" "^2.0.2" - brotli "^1.2.0" - clone "^1.0.4" - deep-equal "^1.0.0" - dfa "^1.2.0" - tiny-inflate "^1.0.2" - unicode-properties "^1.2.2" - unicode-trie "^2.0.0" - -"@foliojs-fork/linebreak@^1.1.1", "@foliojs-fork/linebreak@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@foliojs-fork/linebreak/-/linebreak-1.1.2.tgz#32fee03d5431fa73284373439e172e451ae1e2da" - integrity sha512-ZPohpxxbuKNE0l/5iBJnOAfUaMACwvUIKCvqtWGKIMv1lPYoNjYXRfhi9FeeV9McBkBLxsMFWTVVhHJA8cyzvg== +"@swc/helpers@^0.5.12": + version "0.5.18" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.18.tgz#feeeabea0d10106ee25aaf900165df911ab6d3b1" + integrity sha512-TXTnIcNJQEKwThMMqBXsZ4VGAza6bvN4pa41Rkqoio6QBKMvo+5lexeTMScGCIxtzgQJzElcvIltani+adC5PQ== dependencies: - base64-js "1.3.1" - unicode-trie "^2.0.0" - -"@foliojs-fork/pdfkit@^0.15.3": - version "0.15.3" - resolved "https://registry.yarnpkg.com/@foliojs-fork/pdfkit/-/pdfkit-0.15.3.tgz#590b31e770a98e2af62ce44f268a0d06b41ff32f" - integrity sha512-Obc0Wmy3bm7BINFVvPhcl2rnSSK61DQrlHU8aXnAqDk9LCjWdUOPwhgD8Ywz5VtuFjRxmVOM/kQ/XLIBjDvltw== - dependencies: - "@foliojs-fork/fontkit" "^1.9.2" - "@foliojs-fork/linebreak" "^1.1.1" - crypto-js "^4.2.0" - jpeg-exif "^1.1.4" - png-js "^1.0.0" - -"@foliojs-fork/restructure@^2.0.2": - version "2.0.2" - resolved "https://registry.yarnpkg.com/@foliojs-fork/restructure/-/restructure-2.0.2.tgz#73759aba2aff1da87b7c4554e6839c70d43c92b4" - integrity sha512-59SgoZ3EXbkfSX7b63tsou/SDGzwUEK6MuB5sKqgVK1/XE0fxmpsOb9DQI8LXW3KfGnAjImCGhhEb7uPPAUVNA== + tslib "^2.8.0" "@types/codemirror@^5.60.10": version "5.60.16" @@ -68,10 +37,10 @@ JUMFlot@jumjum123/JUMFlot#*: version "0.0.0" resolved "https://codeload.github.com/jumjum123/JUMFlot/tar.gz/203147fa2ace27db89e2defcde0800654015ae23" -base64-js@1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" - integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== +base64-js@0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-0.0.8.tgz#1101e9544f4a76b1bc3b26d452ca96d7a35e7978" + integrity sha512-3XSA2cR/h/73EzlXXdU6YNycmYI7+kicTxks4eJg2g39biHR84slg2+des+p7iHYhbRg/udIS4TD53WabcOUkw== base64-js@^1.1.2, base64-js@^1.3.0: version "1.5.1" @@ -101,39 +70,13 @@ bootstrap@^3.4.1, bootstrap@~3: resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.4.1.tgz#c3a347d419e289ad11f4033e3c4132b87c081d72" integrity sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA== -brotli@^1.2.0: +brotli@^1.3.2: version "1.3.3" resolved "https://registry.yarnpkg.com/brotli/-/brotli-1.3.3.tgz#7365d8cc00f12cf765d2b2c898716bcf4b604d48" integrity sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg== dependencies: base64-js "^1.1.2" -call-bind-apply-helpers@^1.0.0, call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz#4b5428c222be985d79c3d82657479dbe0b59b2d6" - integrity sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ== - dependencies: - es-errors "^1.3.0" - function-bind "^1.1.2" - -call-bind@^1.0.7, call-bind@^1.0.8: - version "1.0.8" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.8.tgz#0736a9660f537e3388826f440d5ec45f744eaa4c" - integrity sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww== - dependencies: - call-bind-apply-helpers "^1.0.0" - es-define-property "^1.0.0" - get-intrinsic "^1.2.4" - set-function-length "^1.2.2" - -call-bound@^1.0.2: - version "1.0.4" - resolved "https://registry.yarnpkg.com/call-bound/-/call-bound-1.0.4.tgz#238de935d2a2a692928c538c7ccfa91067fd062a" - integrity sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg== - dependencies: - call-bind-apply-helpers "^1.0.2" - get-intrinsic "^1.3.0" - "chosen-bootstrap@https://github.com/dbtek/chosen-bootstrap": version "0.0.0" resolved "https://github.com/dbtek/chosen-bootstrap#12dcd363d1482c54c740ed9fe0e92549d81e9176" @@ -152,10 +95,10 @@ clipboard@^2.0.11: select "^1.1.2" tiny-emitter "^2.0.0" -clone@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" - integrity sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg== +clone@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" + integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w== codemirror-spell-checker@1.1.2: version "1.1.2" @@ -187,19 +130,19 @@ datatables.net-bs@^2: datatables.net "2.3.2" jquery ">=1.7" -datatables.net-buttons-bs@^3.2.5: - version "3.2.5" - resolved "https://registry.yarnpkg.com/datatables.net-buttons-bs/-/datatables.net-buttons-bs-3.2.5.tgz#fe9a8085a66cabd723833834f29e68e91f28ea14" - integrity sha512-7fXOIue+2jpWPWcIrAXWH3BjEhMUD8L2pInT0tqfoEcl/3T+CH0Q6dHJRI5RiYmYKO/HLjpCQ5yqYAL5DT7iHA== +datatables.net-buttons-bs@^3.2.6: + version "3.2.6" + resolved "https://registry.yarnpkg.com/datatables.net-buttons-bs/-/datatables.net-buttons-bs-3.2.6.tgz#8bc20d4df5a539d87add4702fdda134d001b1e77" + integrity sha512-DofCIF9VHbEejmooaVCNxiR7ecrYaWhQN8Tm0sKeofGwYu8h533adUG/SlYy20OSSeF2zRYmY/TsWOJBh5TuAA== dependencies: datatables.net-bs "^2" - datatables.net-buttons "3.2.5" + datatables.net-buttons "3.2.6" jquery ">=1.7" -datatables.net-buttons@3.2.5: - version "3.2.5" - resolved "https://registry.yarnpkg.com/datatables.net-buttons/-/datatables.net-buttons-3.2.5.tgz#e37fc4f06743e057e8e3e4abfda60c988e7c16da" - integrity sha512-OSTl7evbfe0SMee11lyzu5iv/z8Yp05eh3s1QBte/FNqHcoXN8hlAVSSGpYgk5pj8zwHPYIu6fHeMEue4ARUNg== +datatables.net-buttons@3.2.6: + version "3.2.6" + resolved "https://registry.yarnpkg.com/datatables.net-buttons/-/datatables.net-buttons-3.2.6.tgz#dad80c8f28eb18741cec49fb33397073217ca63e" + integrity sha512-rLqkB3xLIAYwVLt+lUSxybo/1WqveTAxhQm6wj6yvXlJiWq+oJ8MKW6H1q90QrXbNp0fGngnfD0cmpMZnNnnNw== dependencies: datatables.net "^2" jquery ">=1.7" @@ -219,43 +162,13 @@ datatables.net@2.3.2: dependencies: jquery ">=1.7" -datatables.net@^2, datatables.net@^2.3.5: - version "2.3.5" - resolved "https://registry.yarnpkg.com/datatables.net/-/datatables.net-2.3.5.tgz#a35cc1209edb7525ea68ebc3e7d3af6e3f30a758" - integrity sha512-Qrwc+vuw8GHo42u1usWTuriNAMW0VvLPSW3j8g3GxvatiD8wS/ZGW32VAYLLfmF4Hz0C/fo2KB3xZBfcpqqVTQ== +datatables.net@^2, datatables.net@^2.3.6: + version "2.3.6" + resolved "https://registry.yarnpkg.com/datatables.net/-/datatables.net-2.3.6.tgz#a11be57a2b50d7231cae2980a8ff1df3c18b7b17" + integrity sha512-xQ/dCxrjfxM0XY70wSIzakkTZ6ghERwlLmAPyCnu8Sk5cyt9YvOVyOsFNOa/BZ/lM63Q3i2YSSvp/o7GXZGsbg== dependencies: jquery ">=1.7" -deep-equal@^1.0.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.1.2.tgz#78a561b7830eef3134c7f6f3a3d6af272a678761" - integrity sha512-5tdhKF6DbU7iIzrIOa1AOUt39ZRm13cmL1cGEh//aqR8x9+tNfbywRf0n5FD/18OKMdo7DNEtrX2t22ZAkI+eg== - dependencies: - is-arguments "^1.1.1" - is-date-object "^1.0.5" - is-regex "^1.1.4" - object-is "^1.1.5" - object-keys "^1.1.1" - regexp.prototype.flags "^1.5.1" - -define-data-property@^1.0.1, define-data-property@^1.1.4: - version "1.1.4" - resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" - integrity sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A== - dependencies: - es-define-property "^1.0.0" - es-errors "^1.3.0" - gopd "^1.0.1" - -define-properties@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.2.1.tgz#10781cc616eb951a80a034bafcaa7377f6af2b6c" - integrity sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg== - dependencies: - define-data-property "^1.0.1" - has-property-descriptors "^1.0.0" - object-keys "^1.1.1" - delegate@^3.1.2: version "3.2.0" resolved "https://registry.yarnpkg.com/delegate/-/delegate-3.2.0.tgz#b66b71c3158522e8ab5744f720d8ca0c2af59166" @@ -280,15 +193,6 @@ drmonty-datatables-responsive@^1.0.0: dependencies: jquery ">=1.7.0" -dunder-proto@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a" - integrity sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A== - dependencies: - call-bind-apply-helpers "^1.0.1" - es-errors "^1.3.0" - gopd "^1.2.0" - easymde@^2.20.0: version "2.20.0" resolved "https://registry.yarnpkg.com/easymde/-/easymde-2.20.0.tgz#88b3161feab6e1900afa9c4dab3f1da352b0a26e" @@ -300,28 +204,16 @@ easymde@^2.20.0: codemirror-spell-checker "1.1.2" marked "^4.1.0" -es-define-property@^1.0.0, es-define-property@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa" - integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g== - -es-errors@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" - integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== - -es-object-atoms@^1.0.0, es-object-atoms@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz#1c4f2c4837327597ce69d2ca190a7fdd172338c1" - integrity sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA== - dependencies: - es-errors "^1.3.0" - eve-raphael@0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/eve-raphael/-/eve-raphael-0.5.0.tgz#17c754b792beef3fa6684d79cf5a47c63c4cda30" integrity sha512-jrxnPsCGqng1UZuEp9DecX/AuSyAszATSjf4oEcRxvfxa1Oux4KkIPKBAAWWnpdwfARtr+Q0o9aPYWjsROD7ug== +fast-deep-equal@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" + integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== + flot@flot/flot#~0.8.3: version "0.8.3" resolved "https://codeload.github.com/flot/flot/tar.gz/453b017cc5acfd75e252b93e8635f57f4196d45d" @@ -336,45 +228,26 @@ font-awesome@~4.4: resolved "https://registry.yarnpkg.com/font-awesome/-/font-awesome-4.4.0.tgz#9fe43f82cf72726badcbdb2704407aadaca17da9" integrity sha512-h45v/TF2b9d6JiBnyluFrjZRyJXXkRjWbTKxb0ygTScxdP8gWdgMBaghbDuSLQFHNkj3M0eNrLxfTEiQo93ARQ== +fontkit@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/fontkit/-/fontkit-2.0.4.tgz#4765d664c68b49b5d6feb6bd1051ee49d8ec5ab0" + integrity sha512-syetQadaUEDNdxdugga9CpEYVaQIxOwk7GlwZWWZ19//qW4zE5bknOKeMBDYAASwnpaSHKJITRLMF9m1fp3s6g== + dependencies: + "@swc/helpers" "^0.5.12" + brotli "^1.3.2" + clone "^2.1.2" + dfa "^1.2.0" + fast-deep-equal "^3.1.3" + restructure "^3.0.0" + tiny-inflate "^1.0.3" + unicode-properties "^1.4.0" + unicode-trie "^2.0.0" + fullcalendar@^3.10.2: version "3.10.5" resolved "https://registry.yarnpkg.com/fullcalendar/-/fullcalendar-3.10.5.tgz#57a3a64d7d744181582bb9e1be32d1846e1db53a" integrity sha512-JGWpECKgza/344bbF5QT0hBJpx04DZ/7QGPlR1ZbAwrG6Yz6mWEkQd+NnZUh1sK6HCBIPnPRW2x53aJxeLGvvQ== -function-bind@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" - integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== - -functions-have-names@^1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834" - integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== - -get-intrinsic@^1.2.4, get-intrinsic@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01" - integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ== - dependencies: - call-bind-apply-helpers "^1.0.2" - es-define-property "^1.0.1" - es-errors "^1.3.0" - es-object-atoms "^1.1.1" - function-bind "^1.1.2" - get-proto "^1.0.1" - gopd "^1.2.0" - has-symbols "^1.1.0" - hasown "^2.0.2" - math-intrinsics "^1.1.0" - -get-proto@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/get-proto/-/get-proto-1.0.1.tgz#150b3f2743869ef3e851ec0c49d15b1d14d00ee1" - integrity sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g== - dependencies: - dunder-proto "^1.0.1" - es-object-atoms "^1.0.0" - good-listener@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/good-listener/-/good-listener-1.2.2.tgz#d53b30cdf9313dffb7dc9a0d477096aa6d145c50" @@ -387,44 +260,6 @@ google-code-prettify@^1.0.0: resolved "https://registry.yarnpkg.com/google-code-prettify/-/google-code-prettify-1.0.5.tgz#9f477f224dbfa62372e5ef803a7e157410400084" integrity sha512-Y47Bw63zJKCuqTuhTZC1ct4e/0ADuMssxXhnrP8QHq71tE2aYBKG6wQwXr8zya0zIUd0mKN3XTlI5AME4qm6NQ== -gopd@^1.0.1, gopd@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" - integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== - -has-property-descriptors@^1.0.0, has-property-descriptors@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854" - integrity sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg== - dependencies: - es-define-property "^1.0.0" - -has-symbols@^1.0.3, has-symbols@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338" - integrity sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ== - -has-tostringtag@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz#2cdc42d40bef2e5b4eeab7c01a73c54ce7ab5abc" - integrity sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw== - dependencies: - has-symbols "^1.0.3" - -hasown@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" - integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== - dependencies: - function-bind "^1.1.2" - -iconv-lite@^0.6.3: - version "0.6.3" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" - integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== - dependencies: - safer-buffer ">= 2.1.2 < 3.0.0" - immediate@~3.0.5: version "3.0.6" resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" @@ -435,32 +270,6 @@ inherits@~2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -is-arguments@^1.1.1: - version "1.2.0" - resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.2.0.tgz#ad58c6aecf563b78ef2bf04df540da8f5d7d8e1b" - integrity sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA== - dependencies: - call-bound "^1.0.2" - has-tostringtag "^1.0.2" - -is-date-object@^1.0.5: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.1.0.tgz#ad85541996fc7aa8b2729701d27b7319f95d82f7" - integrity sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg== - dependencies: - call-bound "^1.0.2" - has-tostringtag "^1.0.2" - -is-regex@^1.1.4: - version "1.2.1" - resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.2.1.tgz#76d70a3ed10ef9be48eb577887d74205bf0cad22" - integrity sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g== - dependencies: - call-bound "^1.0.2" - gopd "^1.2.0" - has-tostringtag "^1.0.2" - hasown "^2.0.2" - isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" @@ -528,16 +337,19 @@ lie@~3.3.0: dependencies: immediate "~3.0.5" +linebreak@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/linebreak/-/linebreak-1.1.0.tgz#831cf378d98bced381d8ab118f852bd50d81e46b" + integrity sha512-MHp03UImeVhB7XZtjd0E4n6+3xr5Dq/9xI/5FptGk5FrbDR3zagPa2DS6U8ks/3HjbKWG9Q1M2ufOzxV2qLYSQ== + dependencies: + base64-js "0.0.8" + unicode-trie "^2.0.0" + marked@^4.1.0: version "4.3.0" resolved "https://registry.yarnpkg.com/marked/-/marked-4.3.0.tgz#796362821b019f734054582038b116481b456cf3" integrity sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A== -math-intrinsics@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" - integrity sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g== - metismenu@~3.0.7: version "3.0.7" resolved "https://registry.yarnpkg.com/metismenu/-/metismenu-3.0.7.tgz#613dd01d14d053474b926a1ecac24d137c934aaa" @@ -552,19 +364,6 @@ morris.js@morrisjs/morris.js: version "0.5.1" resolved "https://codeload.github.com/morrisjs/morris.js/tar.gz/14530d0733801d5bef1264cf3d062ecace7e326b" -object-is@^1.1.5: - version "1.1.6" - resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.6.tgz#1a6a53aed2dd8f7e6775ff870bea58545956ab07" - integrity sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q== - dependencies: - call-bind "^1.0.7" - define-properties "^1.2.1" - -object-keys@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" - integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== - pako@^0.2.5: version "0.2.9" resolved "https://registry.yarnpkg.com/pako/-/pako-0.2.9.tgz#f3f7522f4ef782348da8161bad9ecfd51bf83a75" @@ -575,15 +374,25 @@ pako@~1.0.2: resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== -pdfmake@^0.2.20: - version "0.2.20" - resolved "https://registry.yarnpkg.com/pdfmake/-/pdfmake-0.2.20.tgz#a2e37114e46247c9a295df2fc1c7184942de567e" - integrity sha512-bGbxbGFP5p8PWNT3Phsu1ZcRLnRfF6jmnuKTkgmt6i5PZzSdX6JaB+NeTz9q+aocfW8SE9GUjL3o/5GroBqGcQ== +pdfkit@^0.17.2: + version "0.17.2" + resolved "https://registry.yarnpkg.com/pdfkit/-/pdfkit-0.17.2.tgz#f77d7129ca49bc5015246b890d1aca6cb6b71f00" + integrity sha512-UnwF5fXy08f0dnp4jchFYAROKMNTaPqb/xgR8GtCzIcqoTnbOqtp3bwKvO4688oHI6vzEEs8Q6vqqEnC5IUELw== dependencies: - "@foliojs-fork/linebreak" "^1.1.2" - "@foliojs-fork/pdfkit" "^0.15.3" - iconv-lite "^0.6.3" - xmldoc "^2.0.1" + crypto-js "^4.2.0" + fontkit "^2.0.4" + jpeg-exif "^1.1.4" + linebreak "^1.1.0" + png-js "^1.0.0" + +pdfmake@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/pdfmake/-/pdfmake-0.3.0.tgz#b42c5af9aef4095f1310564d4a2f2d49f950cae9" + integrity sha512-sS7ow3ZrdFjlC7s4J5k3UA5IHQQbXRs6+NtdzfWDR0SvPa7+M8d69rITObFAsJ4t6iwkKRsc87Q+I/gFlTUVQg== + dependencies: + linebreak "^1.1.0" + pdfkit "^0.17.2" + xmldoc "^2.0.3" png-js@^1.0.0: version "1.0.0" @@ -615,60 +424,26 @@ readable-stream@~2.3.6: string_decoder "~1.1.1" util-deprecate "~1.0.1" -regexp.prototype.flags@^1.5.1: - version "1.5.4" - resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz#1ad6c62d44a259007e55b3970e00f746efbcaa19" - integrity sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA== - dependencies: - call-bind "^1.0.8" - define-properties "^1.2.1" - es-errors "^1.3.0" - get-proto "^1.0.1" - gopd "^1.2.0" - set-function-name "^2.0.2" +restructure@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/restructure/-/restructure-3.0.2.tgz#e6b2fad214f78edee21797fa8160fef50eb9b49a" + integrity sha512-gSfoiOEA0VPE6Tukkrr7I0RBdE0s7H1eFCDBk05l1KIQT1UIKNc5JZy6jdyW6eYH3aR3g5b3PuL77rq0hvwtAw== safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -"safer-buffer@>= 2.1.2 < 3.0.0": - version "2.1.2" - resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" - integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== - -sax@^1.2.4: - version "1.4.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" - integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== +sax@^1.4.3: + version "1.4.3" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.3.tgz#fcebae3b756cdc8428321805f4b70f16ec0ab5db" + integrity sha512-yqYn1JhPczigF94DMS+shiDMjDowYO6y9+wB/4WgO0Y19jWYk0lQ4tuG5KI7kj4FTp1wxPj5IFfcrz/s1c3jjQ== select@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/select/-/select-1.1.2.tgz#0e7350acdec80b1108528786ec1d4418d11b396d" integrity sha512-OwpTSOfy6xSs1+pwcNrv0RBMOzI39Lp3qQKUTPVVPRjCdNa5JH/oPRiqsesIskK8TVgmRiHwO4KXlV2Li9dANA== -set-function-length@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" - integrity sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg== - dependencies: - define-data-property "^1.1.4" - es-errors "^1.3.0" - function-bind "^1.1.2" - get-intrinsic "^1.2.4" - gopd "^1.0.1" - has-property-descriptors "^1.0.2" - -set-function-name@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/set-function-name/-/set-function-name-2.0.2.tgz#16a705c5a0dc2f5e638ca96d8a8cd4e1c2b90985" - integrity sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ== - dependencies: - define-data-property "^1.1.4" - es-errors "^1.3.0" - functions-have-names "^1.2.3" - has-property-descriptors "^1.0.2" - setimmediate@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" @@ -691,17 +466,22 @@ tiny-emitter@^2.0.0: resolved "https://registry.yarnpkg.com/tiny-emitter/-/tiny-emitter-2.1.0.tgz#1d1a56edfc51c43e863cbb5382a72330e3555423" integrity sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q== -tiny-inflate@^1.0.0, tiny-inflate@^1.0.2: +tiny-inflate@^1.0.0, tiny-inflate@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/tiny-inflate/-/tiny-inflate-1.0.3.tgz#122715494913a1805166aaf7c93467933eea26c4" integrity sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw== +tslib@^2.8.0: + version "2.8.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" + integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== + typo-js@*: version "1.2.5" resolved "https://registry.yarnpkg.com/typo-js/-/typo-js-1.2.5.tgz#0aa65e0be9b69036463a3827de8185b4144e3086" integrity sha512-F45vFWdGX8xahIk/sOp79z2NJs8ETMYsmMChm9D5Hlx3+9j7VnCyQyvij5MOCrNY3NNe8noSyokRjQRfq+Bc7A== -unicode-properties@^1.2.2: +unicode-properties@^1.4.0: version "1.4.1" resolved "https://registry.yarnpkg.com/unicode-properties/-/unicode-properties-1.4.1.tgz#96a9cffb7e619a0dc7368c28da27e05fc8f9be5f" integrity sha512-CLjCCLQ6UuMxWnbIylkisbRj31qxHPAurvena/0iwSVbQ2G1VY5/HjV0IRabOEbDHlzZlRdCrD4NhB0JtU40Pg== @@ -722,9 +502,9 @@ util-deprecate@~1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== -xmldoc@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/xmldoc/-/xmldoc-2.0.2.tgz#1ad89f9054cc8b1c500135e746da2a608b7bca6b" - integrity sha512-UiRwoSStEXS3R+YE8OqYv3jebza8cBBAI2y8g3B15XFkn3SbEOyyLnmPHjLBPZANrPJKEzxxB7A3XwcLikQVlQ== +xmldoc@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/xmldoc/-/xmldoc-2.0.3.tgz#65b4226b753ea6cd4601f3f56d52338941d38380" + integrity sha512-6gRk4NY/Jvg67xn7OzJuxLRsGgiXBaPUQplVJ/9l99uIugxh4FTOewYz5ic8WScj7Xx/2WvhENiQKwkK9RpE4w== dependencies: - sax "^1.2.4" + sax "^1.4.3" diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index bc31139a352..d3d3a3cd4c1 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -3,7 +3,7 @@ services: uwsgi: build: context: . - dockerfile: Dockerfile.django-debian + dockerfile: Dockerfile.django-${DEFECT_DOJO_OS:-debian} target: development entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-uwsgi-dev.sh'] volumes: diff --git a/docker-compose.override.integration_tests.yml b/docker-compose.override.integration_tests.yml index 215529180df..8d6efe954f7 100644 --- a/docker-compose.override.integration_tests.yml +++ b/docker-compose.override.integration_tests.yml @@ -23,6 +23,8 @@ services: volumes: - defectdojo_media_integration_tests:/usr/share/nginx/html/media uwsgi: + build: + target: development entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-uwsgi-dev.sh'] volumes: - '.:/app:z' diff --git a/docker-compose.yml b/docker-compose.yml index ada66ba1a57..fb566f29611 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -120,7 +120,7 @@ services: source: ./docker/extra_settings target: /app/docker/extra_settings postgres: - image: postgres:18.1-alpine@sha256:154ea39af68ff30dec041cd1f1b5600009993724c811dbadde54126eb10bedd1 + image: postgres:18.1-alpine@sha256:b40d931bd0e7ce6eecc59a5a6ac3b3c04a01e559750e73e7086b6dbd7f8bf545 environment: POSTGRES_DB: ${DD_DATABASE_NAME:-defectdojo} POSTGRES_USER: ${DD_DATABASE_USER:-defectdojo} @@ -128,7 +128,7 @@ services: volumes: - defectdojo_postgres:/var/lib/postgresql/data valkey: - image: valkey/valkey:7.2.11-alpine@sha256:7b2019b47ad58be661fa6eba5ea66106eadde03459387113aaed29a464a5876b + image: valkey/valkey:7.2.11-alpine@sha256:9e483e0fe4c98b631b166b41d530c7ff1b8009a44f261bff28e9d1e2e27db58d volumes: # we keep using the redis volume as renaming is not possible and copying data over # would require steps during downtime or complex commands in the intializer diff --git a/docker/entrypoint-initializer.sh b/docker/entrypoint-initializer.sh index ec193ef6f06..4b01b9c7f37 100755 --- a/docker/entrypoint-initializer.sh +++ b/docker/entrypoint-initializer.sh @@ -5,42 +5,6 @@ set -e # needed to handle "exit" correctly . /secret-file-loader.sh . /reach_database.sh -initialize_data() -{ - # Test types shall be initialized every time by the initializer, to make sure test types are complete - # when new parsers have been implemented - echo "Initialization of test_types" - python3 manage.py initialize_test_types - - # Non-standard permissions cannot be created with a database migration, because the content type will only - # be available after the dojo migrations - echo "Creation of non-standard permissions" - python3 manage.py initialize_permissions -} - -create_announcement_banner() -{ -# Load the announcement banner -if [ -z "$DD_CREATE_CLOUD_BANNER" ]; then -echo "Creating Announcement Banner" -cat </dev/null || true) NUM_FILES=$(echo "$FILES" | wc -w) @@ -56,122 +20,7 @@ fi umask 0002 -if [ "${DD_INITIALIZE}" = false ] -then - echo "Echo initialization skipped. Exiting." - exit -fi -echo "Initializing." - wait_for_database_to_be_reachable -echo - -echo "Checking ENABLE_AUDITLOG" -cat < Import Scan Results`. As the `Scan type`, select the related type (the `API Scan Configuration` created above) and click `Import`. + +## Custom Trust + +In some cases, you may want to connect to a tool that uses a certificate from a certification authority (CA) that is not +in the default trust store (e.g. a company-internal CA), which requires that you add custom trust to an existing trust +store or replace the existing trust store with your own. + +### Using a Custom-Built DefectDojo Image + +When you are building your own container image for `django-DefectDojo`, you can simply add the certificates you would +like to include as custom trust to the `docker/certs` path (see +[Dockerfile.django](https://github.com/DefectDojo/django-DefectDojo/blob/861b617bfcb17cb5e858f46e31509134d0e98171/Dockerfile.django#L70)) + +### Using the Prebuilt DefectDojo Image + +1. Create a new mounted volume where the new trust store will be added (ensures persistence). +2. Create a new trust store + 1. Prepare a new PEM-encoded trust store file (`custom-cacerts.pem`). + 2. Optional, if you want to keep existing trust: Add the custom trust to the existing trust store + 1. Find the location of the existing trust store by running `python -m certifi` in the container + 2. Append your custom trust to the existing trust store by running + `cat cacert.pem custom-cacerts.pem > extended-cacerts.pem`. + ***Important: The consequence of copying the existing trust store is that you will not receive any updates + (added or removed CA certificates).*** +3. Copy the new trust store (`custom-cacerts.pem` or `extended-cacerts.pem`) to the mounted volume. +4. Point the environment variable `REQUESTS_CA_BUNDLE` to the new trust store file. + +> `REQUESTS_CA_BUNDLE` is an environment variable from the Python `requests` package. By default, it uses the trust +> store provided by the `certifi` package. For more details, check the respective documentation +> ([requests](https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification) or +> [certifi](https://certifiio.readthedocs.io/en/latest/)) \ No newline at end of file diff --git a/docs/content/supported_tools/parsers/api/sonarqube.md b/docs/content/supported_tools/parsers/api/sonarqube.md index 3f38e022ebe..c6299ad9f18 100644 --- a/docs/content/supported_tools/parsers/api/sonarqube.md +++ b/docs/content/supported_tools/parsers/api/sonarqube.md @@ -44,13 +44,4 @@ If using a version of SonarQube with multi-branch scanning, the branch to be sca be supplied in the `branch_tag` field at import/re-import time. If the branch does not exist, a notification will be generated in the alerts table, indicating that branch to be imported does not exist. If a branch name is not supplied during import/re-import, the default branch -of the SonarQube project will be used. - -## Custom Trust - -If you are connecting to SonarQube via HTTPS, the issuer of the certificate that is presented by -SonarQube must be trusted. - -One way of achieving this is by defining the `REQUESTS_CA_BUNDLE` environment variable to point -to a PEM-encoded certificate file in the container (e.g. `REQUESTS_CA_BUNDLE=/app/media/cacerts.pem`). -To ensure the certificate is persisted, the file should be in a mounted volume. \ No newline at end of file +of the SonarQube project will be used. \ No newline at end of file diff --git a/docs/content/supported_tools/parsers/file/pingcastle.md b/docs/content/supported_tools/parsers/file/pingcastle.md new file mode 100644 index 00000000000..836877d5519 --- /dev/null +++ b/docs/content/supported_tools/parsers/file/pingcastle.md @@ -0,0 +1,7 @@ +title: "PingCastle" +toc_hide: true +--- +Import results from the [PingCastle](https://www.pingcastle.com/documentation/). + +### Sample Scan Data +Sample PingCastle scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/pingcastle). \ No newline at end of file diff --git a/docs/content/supported_tools/parsers/file/prowler.md b/docs/content/supported_tools/parsers/file/prowler.md new file mode 100644 index 00000000000..5bf1dff2202 --- /dev/null +++ b/docs/content/supported_tools/parsers/file/prowler.md @@ -0,0 +1,14 @@ +--- +title: "Prowler Scan" +toc_hide: true +--- +This parser imports Prowler Scan files in JSON and CSV format. The AWS, GCP, Azure, and Kubernetes could types are supported by this parser. + +### Sample Scan Data +Sample Prowler scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/prowler). + +### Default Deduplication Hashcode Fields +By default, DefectDojo identifies duplicate Findings using these [hashcode fields](https://docs.defectdojo.com/en/working_with_findings/finding_deduplication/about_deduplication/): + +- title +- description diff --git a/docs/layouts/_default/sitemap.xml b/docs/layouts/_default/sitemap.xml new file mode 100644 index 00000000000..1b49b9ec7c1 --- /dev/null +++ b/docs/layouts/_default/sitemap.xml @@ -0,0 +1,13 @@ +{{- /* sitemap for all pages */ -}} +{{ printf "" | safeHTML }} + + {{ range .Site.RegularPages }} + + {{ .Permalink }} + weekly + {{ with .Lastmod }} + {{ .Format "2006-01-02T15:04:05-07:00" | safeHTML }} + {{ end }} + + {{ end }} + diff --git a/docs/package-lock.json b/docs/package-lock.json index aaf7bf14240..6fc5bf14538 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -9,9 +9,9 @@ "version": "1.8.0", "license": "MIT", "dependencies": { - "@docsearch/css": "4.3.2", - "@docsearch/js": "4.3.2", - "@tabler/icons": "3.35.0", + "@docsearch/css": "4.4.0", + "@docsearch/js": "4.4.0", + "@tabler/icons": "3.36.1", "@thulite/doks-core": "1.8.3", "@thulite/images": "3.3.3", "@thulite/inline-svg": "1.2.1", @@ -19,13 +19,311 @@ "thulite": "2.6.3" }, "devDependencies": { - "prettier": "3.7.2", - "vite": "7.2.4" + "prettier": "3.7.4", + "vite": "7.3.0" }, "engines": { "node": ">=20.11.0" } }, + "node_modules/@ai-sdk/gateway": { + "version": "2.0.23", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-2.0.23.tgz", + "integrity": "sha512-qmX7afPRszUqG5hryHF3UN8ITPIRSGmDW6VYCmByzjoUkgm3MekzSx2hMV1wr0P+llDeuXb378SjqUfpvWJulg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.19", + "@vercel/oidc": "3.0.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", + "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "3.0.19", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.19.tgz", + "integrity": "sha512-W41Wc9/jbUVXVwCN/7bWa4IKe8MtxO3EyA0Hfhx6grnmiYlCvpI8neSYWFE0zScXJkgA/YK3BRybzgyiXuu6JA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "2.0.0", + "@standard-schema/spec": "^1.0.0", + "eventsource-parser": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/react": { + "version": "2.0.118", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-2.0.118.tgz", + "integrity": "sha512-K/5VVEGTIu9SWrdQ0s/11OldFU8IjprDzeE6TaC2fOcQWhG7dGVGl9H8Z32QBHzdfJyMhFUxEyFKSOgA2j9+VQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "3.0.19", + "ai": "5.0.116", + "swr": "^2.2.5", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ~19.0.1 || ~19.1.2 || ^19.2.1", + "zod": "^3.25.76 || ^4.1.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@algolia/abtesting": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@algolia/abtesting/-/abtesting-1.12.2.tgz", + "integrity": "sha512-oWknd6wpfNrmRcH0vzed3UPX0i17o4kYLM5OMITyMVM2xLgaRbIafoxL0e8mcrNNb0iORCJA0evnNDKRYth5WQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/autocomplete-core": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-core/-/autocomplete-core-1.19.2.tgz", + "integrity": "sha512-mKv7RyuAzXvwmq+0XRK8HqZXt9iZ5Kkm2huLjgn5JoCPtDy+oh9yxUMfDDaVCw0oyzZ1isdJBc7l9nuCyyR7Nw==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-plugin-algolia-insights": "1.19.2", + "@algolia/autocomplete-shared": "1.19.2" + } + }, + "node_modules/@algolia/autocomplete-plugin-algolia-insights": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-plugin-algolia-insights/-/autocomplete-plugin-algolia-insights-1.19.2.tgz", + "integrity": "sha512-TjxbcC/r4vwmnZaPwrHtkXNeqvlpdyR+oR9Wi2XyfORkiGkLTVhX2j+O9SaCCINbKoDfc+c2PB8NjfOnz7+oKg==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-shared": "1.19.2" + }, + "peerDependencies": { + "search-insights": ">= 1 < 3" + } + }, + "node_modules/@algolia/autocomplete-shared": { + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-shared/-/autocomplete-shared-1.19.2.tgz", + "integrity": "sha512-jEazxZTVD2nLrC+wYlVHQgpBoBB5KPStrJxLzsIFl6Kqd1AlG9sIAGl39V5tECLpIQzB3Qa2T6ZPJ1ChkwMK/w==", + "license": "MIT", + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@algolia/client-abtesting": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/client-abtesting/-/client-abtesting-5.46.2.tgz", + "integrity": "sha512-oRSUHbylGIuxrlzdPA8FPJuwrLLRavOhAmFGgdAvMcX47XsyM+IOGa9tc7/K5SPvBqn4nhppOCEz7BrzOPWc4A==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-analytics": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-5.46.2.tgz", + "integrity": "sha512-EPBN2Oruw0maWOF4OgGPfioTvd+gmiNwx0HmD9IgmlS+l75DatcBkKOPNJN+0z3wBQWUO5oq602ATxIfmTQ8bA==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-common": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.46.2.tgz", + "integrity": "sha512-Hj8gswSJNKZ0oyd0wWissqyasm+wTz1oIsv5ZmLarzOZAp3vFEda8bpDQ8PUhO+DfkbiLyVnAxsPe4cGzWtqkg==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-insights": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/client-insights/-/client-insights-5.46.2.tgz", + "integrity": "sha512-6dBZko2jt8FmQcHCbmNLB0kCV079Mx/DJcySTL3wirgDBUH7xhY1pOuUTLMiGkqM5D8moVZTvTdRKZUJRkrwBA==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-personalization": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-5.46.2.tgz", + "integrity": "sha512-1waE2Uqh/PHNeDXGn/PM/WrmYOBiUGSVxAWqiJIj73jqPqvfzZgzdakHscIVaDl6Cp+j5dwjsZ5LCgaUr6DtmA==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-query-suggestions": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/client-query-suggestions/-/client-query-suggestions-5.46.2.tgz", + "integrity": "sha512-EgOzTZkyDcNL6DV0V/24+oBJ+hKo0wNgyrOX/mePBM9bc9huHxIY2352sXmoZ648JXXY2x//V1kropF/Spx83w==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-search": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-5.46.2.tgz", + "integrity": "sha512-ZsOJqu4HOG5BlvIFnMU0YKjQ9ZI6r3C31dg2jk5kMWPSdhJpYL9xa5hEe7aieE+707dXeMI4ej3diy6mXdZpgA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@algolia/client-common": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/ingestion": { + "version": "1.46.2", + "resolved": "https://registry.npmjs.org/@algolia/ingestion/-/ingestion-1.46.2.tgz", + "integrity": "sha512-1Uw2OslTWiOFDtt83y0bGiErJYy5MizadV0nHnOoHFWMoDqWW0kQoMFI65pXqRSkVvit5zjXSLik2xMiyQJDWQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/monitoring": { + "version": "1.46.2", + "resolved": "https://registry.npmjs.org/@algolia/monitoring/-/monitoring-1.46.2.tgz", + "integrity": "sha512-xk9f+DPtNcddWN6E7n1hyNNsATBCHIqAvVGG2EAGHJc4AFYL18uM/kMTiOKXE/LKDPyy1JhIerrh9oYb7RBrgw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/recommend": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/recommend/-/recommend-5.46.2.tgz", + "integrity": "sha512-NApbTPj9LxGzNw4dYnZmj2BoXiAc8NmbbH6qBNzQgXklGklt/xldTvu+FACN6ltFsTzoNU6j2mWNlHQTKGC5+Q==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/requester-browser-xhr": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-5.46.2.tgz", + "integrity": "sha512-ekotpCwpSp033DIIrsTpYlGUCF6momkgupRV/FA3m62SreTSZUKjgK6VTNyG7TtYfq9YFm/pnh65bATP/ZWJEg==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/requester-fetch": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/requester-fetch/-/requester-fetch-5.46.2.tgz", + "integrity": "sha512-gKE+ZFi/6y7saTr34wS0SqYFDcjHW4Wminv8PDZEi0/mE99+hSrbKgJWxo2ztb5eqGirQTgIh1AMVacGGWM1iw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/requester-node-http": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-5.46.2.tgz", + "integrity": "sha512-ciPihkletp7ttweJ8Zt+GukSVLp2ANJHU+9ttiSxsJZThXc4Y2yJ8HGVWesW5jN1zrsZsezN71KrMx/iZsOYpg==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, "node_modules/@babel/cli": { "version": "7.28.3", "resolved": "https://registry.npmjs.org/@babel/cli/-/cli-7.28.3.tgz", @@ -1481,25 +1779,84 @@ "node": ">=6.9.0" } }, + "node_modules/@docsearch/core": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@docsearch/core/-/core-4.4.0.tgz", + "integrity": "sha512-kiwNo5KEndOnrf5Kq/e5+D9NBMCFgNsDoRpKQJ9o/xnSlheh6b8AXppMuuUVVdAUIhIfQFk/07VLjjk/fYyKmw==", + "license": "MIT", + "peerDependencies": { + "@types/react": ">= 16.8.0 < 20.0.0", + "react": ">= 16.8.0 < 20.0.0", + "react-dom": ">= 16.8.0 < 20.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, "node_modules/@docsearch/css": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/@docsearch/css/-/css-4.3.2.tgz", - "integrity": "sha512-K3Yhay9MgkBjJJ0WEL5MxnACModX9xuNt3UlQQkDEDZJZ0+aeWKtOkxHNndMRkMBnHdYvQjxkm6mdlneOtU1IQ==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@docsearch/css/-/css-4.4.0.tgz", + "integrity": "sha512-e9vPgtih6fkawakmYo0Y6V4BKBmDV7Ykudn7ADWXUs5b6pmtBRwDbpSG/WiaUG63G28OkJDEnsMvgIAnZgGwYw==", "license": "MIT" }, "node_modules/@docsearch/js": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/@docsearch/js/-/js-4.3.2.tgz", - "integrity": "sha512-xdfpPXMgKRY9EW7U1vtY7gLKbLZFa9ed+t0Dacquq8zXBqAlH9HlUf0h4Mhxm0xatsVeMaIR2wr/u6g0GsZyQw==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@docsearch/js/-/js-4.4.0.tgz", + "integrity": "sha512-vCiKzjYD54bugUIMZA6YzuLDilkD3TNH/kfbvqsnzxiLTMu8F13psD+hdMSEOn7j+dFJOaf49fZ+gwr+rXctMw==", "license": "MIT", "dependencies": { + "@docsearch/react": "4.4.0", "htm": "3.1.1" } }, + "node_modules/@docsearch/react": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@docsearch/react/-/react-4.4.0.tgz", + "integrity": "sha512-z12zeg1mV7WD4Ag4pKSuGukETJLaucVFwszDXL/qLaEgRqxEaVacO9SR1qqnCXvZztlvz2rt7cMqryi/7sKfjA==", + "license": "MIT", + "dependencies": { + "@ai-sdk/react": "^2.0.30", + "@algolia/autocomplete-core": "1.19.2", + "@docsearch/core": "4.4.0", + "@docsearch/css": "4.4.0", + "ai": "^5.0.30", + "algoliasearch": "^5.28.0", + "marked": "^16.3.0", + "zod": "^4.1.8" + }, + "peerDependencies": { + "@types/react": ">= 16.8.0 < 20.0.0", + "react": ">= 16.8.0 < 20.0.0", + "react-dom": ">= 16.8.0 < 20.0.0", + "search-insights": ">= 1 < 3" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + }, + "search-insights": { + "optional": true + } + } + }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.11.tgz", - "integrity": "sha512-Xt1dOL13m8u0WE8iplx9Ibbm+hFAO0GsU2P34UNoDGvZYkY8ifSiy6Zuc1lYxfG7svWE2fzqCUmFp5HCn51gJg==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.1.tgz", + "integrity": "sha512-HHB50pdsBX6k47S4u5g/CaLjqS3qwaOVE5ILsq64jyzgMhLuCuZ8rGzM9yhsAjfjkbgUPMzZEPa7DAp7yz6vuA==", "cpu": [ "ppc64" ], @@ -1514,9 +1871,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.11.tgz", - "integrity": "sha512-uoa7dU+Dt3HYsethkJ1k6Z9YdcHjTrSb5NUy66ZfZaSV8hEYGD5ZHbEMXnqLFlbBflLsl89Zke7CAdDJ4JI+Gg==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.1.tgz", + "integrity": "sha512-kFqa6/UcaTbGm/NncN9kzVOODjhZW8e+FRdSeypWe6j33gzclHtwlANs26JrupOntlcWmB0u8+8HZo8s7thHvg==", "cpu": [ "arm" ], @@ -1531,9 +1888,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.11.tgz", - "integrity": "sha512-9slpyFBc4FPPz48+f6jyiXOx/Y4v34TUeDDXJpZqAWQn/08lKGeD8aDp9TMn9jDz2CiEuHwfhRmGBvpnd/PWIQ==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.1.tgz", + "integrity": "sha512-45fuKmAJpxnQWixOGCrS+ro4Uvb4Re9+UTieUY2f8AEc+t7d4AaZ6eUJ3Hva7dtrxAAWHtlEFsXFMAgNnGU9uQ==", "cpu": [ "arm64" ], @@ -1548,9 +1905,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.11.tgz", - "integrity": "sha512-Sgiab4xBjPU1QoPEIqS3Xx+R2lezu0LKIEcYe6pftr56PqPygbB7+szVnzoShbx64MUupqoE0KyRlN7gezbl8g==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.1.tgz", + "integrity": "sha512-LBEpOz0BsgMEeHgenf5aqmn/lLNTFXVfoWMUox8CtWWYK9X4jmQzWjoGoNb8lmAYml/tQ/Ysvm8q7szu7BoxRQ==", "cpu": [ "x64" ], @@ -1565,9 +1922,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.11.tgz", - "integrity": "sha512-VekY0PBCukppoQrycFxUqkCojnTQhdec0vevUL/EDOCnXd9LKWqD/bHwMPzigIJXPhC59Vd1WFIL57SKs2mg4w==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.1.tgz", + "integrity": "sha512-veg7fL8eMSCVKL7IW4pxb54QERtedFDfY/ASrumK/SbFsXnRazxY4YykN/THYqFnFwJ0aVjiUrVG2PwcdAEqQQ==", "cpu": [ "arm64" ], @@ -1582,9 +1939,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.11.tgz", - "integrity": "sha512-+hfp3yfBalNEpTGp9loYgbknjR695HkqtY3d3/JjSRUyPg/xd6q+mQqIb5qdywnDxRZykIHs3axEqU6l1+oWEQ==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.1.tgz", + "integrity": "sha512-+3ELd+nTzhfWb07Vol7EZ+5PTbJ/u74nC6iv4/lwIU99Ip5uuY6QoIf0Hn4m2HoV0qcnRivN3KSqc+FyCHjoVQ==", "cpu": [ "x64" ], @@ -1599,9 +1956,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.11.tgz", - "integrity": "sha512-CmKjrnayyTJF2eVuO//uSjl/K3KsMIeYeyN7FyDBjsR3lnSJHaXlVoAK8DZa7lXWChbuOk7NjAc7ygAwrnPBhA==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.1.tgz", + "integrity": "sha512-/8Rfgns4XD9XOSXlzUDepG8PX+AVWHliYlUkFI3K3GB6tqbdjYqdhcb4BKRd7C0BhZSoaCxhv8kTcBrcZWP+xg==", "cpu": [ "arm64" ], @@ -1616,9 +1973,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.11.tgz", - "integrity": "sha512-Dyq+5oscTJvMaYPvW3x3FLpi2+gSZTCE/1ffdwuM6G1ARang/mb3jvjxs0mw6n3Lsw84ocfo9CrNMqc5lTfGOw==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.1.tgz", + "integrity": "sha512-GITpD8dK9C+r+5yRT/UKVT36h/DQLOHdwGVwwoHidlnA168oD3uxA878XloXebK4Ul3gDBBIvEdL7go9gCUFzQ==", "cpu": [ "x64" ], @@ -1633,9 +1990,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.11.tgz", - "integrity": "sha512-TBMv6B4kCfrGJ8cUPo7vd6NECZH/8hPpBHHlYI3qzoYFvWu2AdTvZNuU/7hsbKWqu/COU7NIK12dHAAqBLLXgw==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.1.tgz", + "integrity": "sha512-ieMID0JRZY/ZeCrsFQ3Y3NlHNCqIhTprJfDgSB3/lv5jJZ8FX3hqPyXWhe+gvS5ARMBJ242PM+VNz/ctNj//eA==", "cpu": [ "arm" ], @@ -1650,9 +2007,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.11.tgz", - "integrity": "sha512-Qr8AzcplUhGvdyUF08A1kHU3Vr2O88xxP0Tm8GcdVOUm25XYcMPp2YqSVHbLuXzYQMf9Bh/iKx7YPqECs6ffLA==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.1.tgz", + "integrity": "sha512-W9//kCrh/6in9rWIBdKaMtuTTzNj6jSeG/haWBADqLLa9P8O5YSRDzgD5y9QBok4AYlzS6ARHifAb75V6G670Q==", "cpu": [ "arm64" ], @@ -1667,9 +2024,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.11.tgz", - "integrity": "sha512-TmnJg8BMGPehs5JKrCLqyWTVAvielc615jbkOirATQvWWB1NMXY77oLMzsUjRLa0+ngecEmDGqt5jiDC6bfvOw==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.1.tgz", + "integrity": "sha512-VIUV4z8GD8rtSVMfAj1aXFahsi/+tcoXXNYmXgzISL+KB381vbSTNdeZHHHIYqFyXcoEhu9n5cT+05tRv13rlw==", "cpu": [ "ia32" ], @@ -1684,9 +2041,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.11.tgz", - "integrity": "sha512-DIGXL2+gvDaXlaq8xruNXUJdT5tF+SBbJQKbWy/0J7OhU8gOHOzKmGIlfTTl6nHaCOoipxQbuJi7O++ldrxgMw==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.1.tgz", + "integrity": "sha512-l4rfiiJRN7sTNI//ff65zJ9z8U+k6zcCg0LALU5iEWzY+a1mVZ8iWC1k5EsNKThZ7XCQ6YWtsZ8EWYm7r1UEsg==", "cpu": [ "loong64" ], @@ -1701,9 +2058,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.11.tgz", - "integrity": "sha512-Osx1nALUJu4pU43o9OyjSCXokFkFbyzjXb6VhGIJZQ5JZi8ylCQ9/LFagolPsHtgw6himDSyb5ETSfmp4rpiKQ==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.1.tgz", + "integrity": "sha512-U0bEuAOLvO/DWFdygTHWY8C067FXz+UbzKgxYhXC0fDieFa0kDIra1FAhsAARRJbvEyso8aAqvPdNxzWuStBnA==", "cpu": [ "mips64el" ], @@ -1718,9 +2075,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.11.tgz", - "integrity": "sha512-nbLFgsQQEsBa8XSgSTSlrnBSrpoWh7ioFDUmwo158gIm5NNP+17IYmNWzaIzWmgCxq56vfr34xGkOcZ7jX6CPw==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.1.tgz", + "integrity": "sha512-NzdQ/Xwu6vPSf/GkdmRNsOfIeSGnh7muundsWItmBsVpMoNPVpM61qNzAVY3pZ1glzzAxLR40UyYM23eaDDbYQ==", "cpu": [ "ppc64" ], @@ -1735,9 +2092,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.11.tgz", - "integrity": "sha512-HfyAmqZi9uBAbgKYP1yGuI7tSREXwIb438q0nqvlpxAOs3XnZ8RsisRfmVsgV486NdjD7Mw2UrFSw51lzUk1ww==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.1.tgz", + "integrity": "sha512-7zlw8p3IApcsN7mFw0O1Z1PyEk6PlKMu18roImfl3iQHTnr/yAfYv6s4hXPidbDoI2Q0pW+5xeoM4eTCC0UdrQ==", "cpu": [ "riscv64" ], @@ -1752,9 +2109,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.11.tgz", - "integrity": "sha512-HjLqVgSSYnVXRisyfmzsH6mXqyvj0SA7pG5g+9W7ESgwA70AXYNpfKBqh1KbTxmQVaYxpzA/SvlB9oclGPbApw==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.1.tgz", + "integrity": "sha512-cGj5wli+G+nkVQdZo3+7FDKC25Uh4ZVwOAK6A06Hsvgr8WqBBuOy/1s+PUEd/6Je+vjfm6stX0kmib5b/O2Ykw==", "cpu": [ "s390x" ], @@ -1769,9 +2126,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.11.tgz", - "integrity": "sha512-HSFAT4+WYjIhrHxKBwGmOOSpphjYkcswF449j6EjsjbinTZbp8PJtjsVK1XFJStdzXdy/jaddAep2FGY+wyFAQ==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.1.tgz", + "integrity": "sha512-z3H/HYI9MM0HTv3hQZ81f+AKb+yEoCRlUby1F80vbQ5XdzEMyY/9iNlAmhqiBKw4MJXwfgsh7ERGEOhrM1niMA==", "cpu": [ "x64" ], @@ -1786,9 +2143,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.11.tgz", - "integrity": "sha512-hr9Oxj1Fa4r04dNpWr3P8QKVVsjQhqrMSUzZzf+LZcYjZNqhA3IAfPQdEh1FLVUJSiu6sgAwp3OmwBfbFgG2Xg==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.1.tgz", + "integrity": "sha512-wzC24DxAvk8Em01YmVXyjl96Mr+ecTPyOuADAvjGg+fyBpGmxmcr2E5ttf7Im8D0sXZihpxzO1isus8MdjMCXQ==", "cpu": [ "arm64" ], @@ -1803,9 +2160,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.11.tgz", - "integrity": "sha512-u7tKA+qbzBydyj0vgpu+5h5AeudxOAGncb8N6C9Kh1N4n7wU1Xw1JDApsRjpShRpXRQlJLb9wY28ELpwdPcZ7A==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.1.tgz", + "integrity": "sha512-1YQ8ybGi2yIXswu6eNzJsrYIGFpnlzEWRl6iR5gMgmsrR0FcNoV1m9k9sc3PuP5rUBLshOZylc9nqSgymI+TYg==", "cpu": [ "x64" ], @@ -1820,9 +2177,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.11.tgz", - "integrity": "sha512-Qq6YHhayieor3DxFOoYM1q0q1uMFYb7cSpLD2qzDSvK1NAvqFi8Xgivv0cFC6J+hWVw2teCYltyy9/m/14ryHg==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.1.tgz", + "integrity": "sha512-5Z+DzLCrq5wmU7RDaMDe2DVXMRm2tTDvX2KU14JJVBN2CT/qov7XVix85QoJqHltpvAOZUAc3ndU56HSMWrv8g==", "cpu": [ "arm64" ], @@ -1837,9 +2194,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.11.tgz", - "integrity": "sha512-CN+7c++kkbrckTOz5hrehxWN7uIhFFlmS/hqziSFVWpAzpWrQoAG4chH+nN3Be+Kzv/uuo7zhX716x3Sn2Jduw==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.1.tgz", + "integrity": "sha512-Q73ENzIdPF5jap4wqLtsfh8YbYSZ8Q0wnxplOlZUOyZy7B4ZKW8DXGWgTCZmF8VWD7Tciwv5F4NsRf6vYlZtqg==", "cpu": [ "x64" ], @@ -1854,9 +2211,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.11.tgz", - "integrity": "sha512-rOREuNIQgaiR+9QuNkbkxubbp8MSO9rONmwP5nKncnWJ9v5jQ4JxFnLu4zDSRPf3x4u+2VN4pM4RdyIzDty/wQ==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.1.tgz", + "integrity": "sha512-ajbHrGM/XiK+sXM0JzEbJAen+0E+JMQZ2l4RR4VFwvV9JEERx+oxtgkpoKv1SevhjavK2z2ReHk32pjzktWbGg==", "cpu": [ "arm64" ], @@ -1871,9 +2228,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.11.tgz", - "integrity": "sha512-nq2xdYaWxyg9DcIyXkZhcYulC6pQ2FuCgem3LI92IwMgIZ69KHeY8T4Y88pcwoLIjbed8n36CyKoYRDygNSGhA==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.1.tgz", + "integrity": "sha512-IPUW+y4VIjuDVn+OMzHc5FV4GubIwPnsz6ubkvN8cuhEqH81NovB53IUlrlBkPMEPxvNnf79MGBoz8rZ2iW8HA==", "cpu": [ "x64" ], @@ -1888,9 +2245,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.11.tgz", - "integrity": "sha512-3XxECOWJq1qMZ3MN8srCJ/QfoLpL+VaxD/WfNRm1O3B4+AZ/BnLVgFbUV3eiRYDMXetciH16dwPbbHqwe1uU0Q==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.1.tgz", + "integrity": "sha512-RIVRWiljWA6CdVu8zkWcRmGP7iRRIIwvhDKem8UMBjPql2TXM5PkDVvvrzMtj1V+WFPB4K7zkIGM7VzRtFkjdg==", "cpu": [ "arm64" ], @@ -1905,9 +2262,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.11.tgz", - "integrity": "sha512-3ukss6gb9XZ8TlRyJlgLn17ecsK4NSQTmdIXRASVsiS2sQ6zPPZklNJT5GR5tE/MUarymmy8kCEf5xPCNCqVOA==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.1.tgz", + "integrity": "sha512-2BR5M8CPbptC1AK5JbJT1fWrHLvejwZidKx3UMSF0ecHMa+smhi16drIrCEggkgviBwLYd5nwrFLSl5Kho96RQ==", "cpu": [ "ia32" ], @@ -1922,9 +2279,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.11.tgz", - "integrity": "sha512-D7Hpz6A2L4hzsRpPaCYkQnGOotdUpDzSGRIv9I+1ITdHROSFUWW95ZPZWQmGka1Fg7W3zFJowyn9WGwMJ0+KPA==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.1.tgz", + "integrity": "sha512-d5X6RMYv6taIymSk8JBP+nxv8DQAMY6A51GPgusqLdK9wBz5wWIXy1KjTck6HnjE9hqJzJRdk+1p/t5soSbCtw==", "cpu": [ "x64" ], @@ -2119,6 +2476,15 @@ "license": "MIT", "optional": true }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@popperjs/core": { "version": "2.11.8", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", @@ -2438,10 +2804,16 @@ "win32" ] }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, "node_modules/@tabler/icons": { - "version": "3.35.0", - "resolved": "https://registry.npmjs.org/@tabler/icons/-/icons-3.35.0.tgz", - "integrity": "sha512-yYXe+gJ56xlZFiXwV9zVoe3FWCGuZ/D7/G4ZIlDtGxSx5CGQK110wrnT29gUj52kEZoxqF7oURTk97GQxELOFQ==", + "version": "3.36.1", + "resolved": "https://registry.npmjs.org/@tabler/icons/-/icons-3.36.1.tgz", + "integrity": "sha512-f4Jg3Fof/Vru5ioix/UO4GX+sdDsF9wQo47FbtvG+utIYYVQ/QVAC0QYgcBbAjQGfbdOh2CCf0BgiFOF9Ixtjw==", "license": "MIT", "funding": { "type": "github", @@ -2531,6 +2903,59 @@ "dev": true, "license": "MIT" }, + "node_modules/@vercel/oidc": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.0.5.tgz", + "integrity": "sha512-fnYhv671l+eTTp48gB4zEsTW/YtRgRPnkI2nT7x6qw5rkI1Lq2hTmQIpHPgyThI0znLK+vX2n9XxKdXZ7BUbbw==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/ai": { + "version": "5.0.116", + "resolved": "https://registry.npmjs.org/ai/-/ai-5.0.116.tgz", + "integrity": "sha512-+2hYJ80/NcDWuv9K2/MLP3cTCFgwWHmHlS1tOpFUKKcmLbErAAlE/S2knsKboc3PNAu8pQkDr2N3K/Vle7ENgQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "2.0.23", + "@ai-sdk/provider": "2.0.0", + "@ai-sdk/provider-utils": "3.0.19", + "@opentelemetry/api": "1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/algoliasearch": { + "version": "5.46.2", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-5.46.2.tgz", + "integrity": "sha512-qqAXW9QvKf2tTyhpDA4qXv1IfBwD2eduSW6tUEBFIfCeE9gn9HQ9I5+MaKoenRuHrzk5sQoNh1/iof8mY7uD6Q==", + "license": "MIT", + "peer": true, + "dependencies": { + "@algolia/abtesting": "1.12.2", + "@algolia/client-abtesting": "5.46.2", + "@algolia/client-analytics": "5.46.2", + "@algolia/client-common": "5.46.2", + "@algolia/client-insights": "5.46.2", + "@algolia/client-personalization": "5.46.2", + "@algolia/client-query-suggestions": "5.46.2", + "@algolia/client-search": "5.46.2", + "@algolia/ingestion": "1.46.2", + "@algolia/monitoring": "1.46.2", + "@algolia/recommend": "5.46.2", + "@algolia/requester-browser-xhr": "5.46.2", + "@algolia/requester-fetch": "5.46.2", + "@algolia/requester-node-http": "5.46.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -2943,6 +3368,15 @@ "node": ">=4" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -2962,9 +3396,9 @@ "license": "MIT" }, "node_modules/esbuild": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.11.tgz", - "integrity": "sha512-KohQwyzrKTQmhXDW1PjCv3Tyspn9n5GcY2RTDqeORIdIJY8yKIF7sTSopFmn/wpMPW4rdPXI0UE5LJLuq3bx0Q==", + "version": "0.27.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.1.tgz", + "integrity": "sha512-yY35KZckJJuVVPXpvjgxiCuVEJT67F6zDeVTv4rizyPrfGBUpZQsvmxnN+C371c2esD/hNMjj4tpBhuueLN7aA==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -2975,32 +3409,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.11", - "@esbuild/android-arm": "0.25.11", - "@esbuild/android-arm64": "0.25.11", - "@esbuild/android-x64": "0.25.11", - "@esbuild/darwin-arm64": "0.25.11", - "@esbuild/darwin-x64": "0.25.11", - "@esbuild/freebsd-arm64": "0.25.11", - "@esbuild/freebsd-x64": "0.25.11", - "@esbuild/linux-arm": "0.25.11", - "@esbuild/linux-arm64": "0.25.11", - "@esbuild/linux-ia32": "0.25.11", - "@esbuild/linux-loong64": "0.25.11", - "@esbuild/linux-mips64el": "0.25.11", - "@esbuild/linux-ppc64": "0.25.11", - "@esbuild/linux-riscv64": "0.25.11", - "@esbuild/linux-s390x": "0.25.11", - "@esbuild/linux-x64": "0.25.11", - "@esbuild/netbsd-arm64": "0.25.11", - "@esbuild/netbsd-x64": "0.25.11", - "@esbuild/openbsd-arm64": "0.25.11", - "@esbuild/openbsd-x64": "0.25.11", - "@esbuild/openharmony-arm64": "0.25.11", - "@esbuild/sunos-x64": "0.25.11", - "@esbuild/win32-arm64": "0.25.11", - "@esbuild/win32-ia32": "0.25.11", - "@esbuild/win32-x64": "0.25.11" + "@esbuild/aix-ppc64": "0.27.1", + "@esbuild/android-arm": "0.27.1", + "@esbuild/android-arm64": "0.27.1", + "@esbuild/android-x64": "0.27.1", + "@esbuild/darwin-arm64": "0.27.1", + "@esbuild/darwin-x64": "0.27.1", + "@esbuild/freebsd-arm64": "0.27.1", + "@esbuild/freebsd-x64": "0.27.1", + "@esbuild/linux-arm": "0.27.1", + "@esbuild/linux-arm64": "0.27.1", + "@esbuild/linux-ia32": "0.27.1", + "@esbuild/linux-loong64": "0.27.1", + "@esbuild/linux-mips64el": "0.27.1", + "@esbuild/linux-ppc64": "0.27.1", + "@esbuild/linux-riscv64": "0.27.1", + "@esbuild/linux-s390x": "0.27.1", + "@esbuild/linux-x64": "0.27.1", + "@esbuild/netbsd-arm64": "0.27.1", + "@esbuild/netbsd-x64": "0.27.1", + "@esbuild/openbsd-arm64": "0.27.1", + "@esbuild/openbsd-x64": "0.27.1", + "@esbuild/openharmony-arm64": "0.27.1", + "@esbuild/sunos-x64": "0.27.1", + "@esbuild/win32-arm64": "0.27.1", + "@esbuild/win32-ia32": "0.27.1", + "@esbuild/win32-x64": "0.27.1" } }, "node_modules/escalade": { @@ -3021,6 +3455,15 @@ "node": ">=0.10.0" } }, + "node_modules/eventsource-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -3461,6 +3904,12 @@ "node": ">=6" } }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -3570,6 +4019,18 @@ "semver": "bin/semver" } }, + "node_modules/marked": { + "version": "16.4.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz", + "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -3944,9 +4405,9 @@ "license": "MIT" }, "node_modules/prettier": { - "version": "3.7.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.7.2.tgz", - "integrity": "sha512-n3HV2J6QhItCXndGa3oMWvWFAgN1ibnS7R9mt6iokScBOC0Ul9/iZORmU2IWUMcyAQaMPjTlY3uT34TqocUxMA==", + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.7.4.tgz", + "integrity": "sha512-v6UNi1+3hSlVvv8fSaoUbggEM5VErKmmpGA7Pl3HF8V6uKY7rvClBOJlH6yNwQtfTueNkGVpOv/mtWL9L4bgRA==", "dev": true, "license": "MIT", "bin": { @@ -4063,6 +4524,16 @@ } } }, + "node_modules/react": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", + "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/read-cache": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", @@ -4257,6 +4728,13 @@ "node": ">=6.0.0" } }, + "node_modules/search-insights": { + "version": "2.17.3", + "resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz", + "integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==", + "license": "MIT", + "peer": true + }, "node_modules/select": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/select/-/select-1.1.2.tgz", @@ -4395,12 +4873,37 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/swr": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.3.8.tgz", + "integrity": "sha512-gaCPRVoMq8WGDcWj9p4YWzCMPHzE0WNl6W8ADIx9c3JBEIdMkJGMzW+uzXvxHMltwcYACr9jP+32H8/hgwMR7w==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/thenby": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/thenby/-/thenby-1.3.4.tgz", "integrity": "sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ==", "license": "Apache-2.0" }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/throttles": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/throttles/-/throttles-1.0.1.tgz", @@ -4565,6 +5068,15 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/use-sync-external-store": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", + "integrity": "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -4572,13 +5084,13 @@ "license": "MIT" }, "node_modules/vite": { - "version": "7.2.4", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.2.4.tgz", - "integrity": "sha512-NL8jTlbo0Tn4dUEXEsUg8KeyG/Lkmc4Fnzb8JXN/Ykm9G4HNImjtABMJgkQoVjOBN/j2WAwDTRytdqJbZsah7w==", + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.0.tgz", + "integrity": "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "^0.25.0", + "esbuild": "^0.27.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", @@ -4793,6 +5305,16 @@ "engines": { "node": ">=12" } + }, + "node_modules/zod": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.2.1.tgz", + "integrity": "sha512-0wZ1IRqGGhMP76gLqz8EyfBXKk0J2qo2+H3fi4mcUP/KtTocoX08nmIAHl1Z2kJIZbZee8KOpBCSNPRgauucjw==", + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/docs/package.json b/docs/package.json index 34632cb1913..30d36a2f746 100644 --- a/docs/package.json +++ b/docs/package.json @@ -16,9 +16,9 @@ "preview": "vite preview --outDir public" }, "dependencies": { - "@docsearch/css": "4.3.2", - "@docsearch/js": "4.3.2", - "@tabler/icons": "3.35.0", + "@docsearch/css": "4.4.0", + "@docsearch/js": "4.4.0", + "@tabler/icons": "3.36.1", "@thulite/doks-core": "1.8.3", "@thulite/images": "3.3.3", "@thulite/inline-svg": "1.2.1", @@ -26,8 +26,8 @@ "thulite": "2.6.3" }, "devDependencies": { - "prettier": "3.7.2", - "vite": "7.2.4" + "prettier": "3.7.4", + "vite": "7.3.0" }, "engines": { "node": ">=20.11.0" diff --git a/dojo/__init__.py b/dojo/__init__.py index f31e0202149..af8f028cb1c 100644 --- a/dojo/__init__.py +++ b/dojo/__init__.py @@ -4,6 +4,6 @@ # Django starts so that shared_task will use this app. from .celery import app as celery_app # noqa: F401 -__version__ = "2.53.5" +__version__ = "2.54.0" __url__ = "https://github.com/DefectDojo/django-DefectDojo" __docs__ = "https://documentation.defectdojo.com" diff --git a/dojo/api_v2/permissions.py b/dojo/api_v2/permissions.py index a0a25e5364f..421fb87b526 100644 --- a/dojo/api_v2/permissions.py +++ b/dojo/api_v2/permissions.py @@ -12,6 +12,7 @@ user_has_configuration_permission, user_has_global_permission, user_has_permission, + user_is_superuser_or_global_owner, ) from dojo.authorization.roles_permissions import Permissions from dojo.importers.auto_create_context import AutoCreateContextManager @@ -872,6 +873,11 @@ def has_permission(self, request, view): return request.user and request.user.is_superuser +class IsSuperUserOrGlobalOwner(permissions.BasePermission): + def has_permission(self, request, view): + return user_is_superuser_or_global_owner(request.user) + + class UserHasEngagementPresetPermission(permissions.BasePermission): def has_permission(self, request, view): return check_post_permission( diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index 4cada0d1dc6..6bc18b5115f 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -29,6 +29,7 @@ from dojo.authorization.roles_permissions import Permissions from dojo.endpoint.utils import endpoint_filter, endpoint_meta_import from dojo.finding.helper import ( + save_endpoints_template, save_vulnerability_ids, save_vulnerability_ids_template, ) @@ -111,7 +112,6 @@ User, UserContactInfo, Vulnerability_Id, - Vulnerability_Id_Template, get_current_date, ) from dojo.notifications.helper import create_notification @@ -487,6 +487,8 @@ class UserSerializer(serializers.ModelSerializer): date_joined = serializers.DateTimeField(read_only=True) last_login = serializers.DateTimeField(read_only=True, allow_null=True) email = serializers.EmailField(required=True) + token_last_reset = serializers.SerializerMethodField() + password_last_reset = serializers.SerializerMethodField() password = serializers.CharField( write_only=True, style={"input_type": "password"}, @@ -515,10 +517,22 @@ class Meta: "last_login", "is_active", "is_superuser", + "token_last_reset", + "password_last_reset", "password", "configuration_permissions", ) + @extend_schema_field(serializers.DateTimeField(allow_null=True)) + def get_token_last_reset(self, instance): + uci = getattr(instance, "usercontactinfo", None) + return getattr(uci, "token_last_reset", None) + + @extend_schema_field(serializers.DateTimeField(allow_null=True)) + def get_password_last_reset(self, instance): + uci = getattr(instance, "usercontactinfo", None) + return getattr(uci, "password_last_reset", None) + def to_representation(self, instance): ret = super().to_representation(instance) @@ -2015,57 +2029,80 @@ def validate_severity(self, value: str) -> str: return value -class VulnerabilityIdTemplateSerializer(serializers.ModelSerializer): - class Meta: - model = Vulnerability_Id_Template - fields = ["vulnerability_id"] - - class FindingTemplateSerializer(serializers.ModelSerializer): tags = TagListSerializerField(required=False) - vulnerability_ids = VulnerabilityIdTemplateSerializer( - source="vulnerability_id_template_set", many=True, required=False, - ) + vulnerability_ids = serializers.SerializerMethodField() + endpoints = serializers.SerializerMethodField() class Meta: model = Finding_Template - exclude = ("cve",) + exclude = ("cve", "vulnerability_ids_text") + + @extend_schema_field(serializers.ListField(child=serializers.CharField())) + def get_vulnerability_ids(self, obj): + """Return vulnerability IDs as a list of strings.""" + return obj.vulnerability_ids + + @extend_schema_field(serializers.ListField(child=serializers.CharField())) + def get_endpoints(self, obj): + """Return endpoints as a list of URL strings.""" + return obj.endpoints if hasattr(obj, "endpoints") else [] def create(self, validated_data): - # Save vulnerability ids and pop them - if "vulnerability_id_template_set" in validated_data: - vulnerability_id_set = validated_data.pop( - "vulnerability_id_template_set", - ) - else: - vulnerability_id_set = None + # Handle vulnerability_ids if provided as list + vulnerability_ids = None + if "vulnerability_ids" in self.initial_data: + vulnerability_ids = self.initial_data.get("vulnerability_ids", []) + if isinstance(vulnerability_ids, str): + # If it's a string, split by newlines + vulnerability_ids = [vid.strip() for vid in vulnerability_ids.split("\n") if vid.strip()] + elif not isinstance(vulnerability_ids, list): + vulnerability_ids = [] + + # Handle endpoints if provided as list + endpoint_urls = None + if "endpoints" in self.initial_data: + endpoint_urls = self.initial_data.get("endpoints", []) + if isinstance(endpoint_urls, str): + # If it's a string, split by newlines + endpoint_urls = [url.strip() for url in endpoint_urls.split("\n") if url.strip()] + elif not isinstance(endpoint_urls, list): + endpoint_urls = [] new_finding_template = super().create( validated_data, ) - if vulnerability_id_set: - vulnerability_ids = [vulnerability_id["vulnerability_id"] for vulnerability_id in vulnerability_id_set] - validated_data["cve"] = vulnerability_ids[0] - save_vulnerability_ids_template( - new_finding_template, vulnerability_ids, - ) - new_finding_template.save() + # Save vulnerability IDs using helper + if vulnerability_ids: + save_vulnerability_ids_template(new_finding_template, vulnerability_ids) + + # Save endpoints using helper + if endpoint_urls: + save_endpoints_template(new_finding_template, endpoint_urls) return new_finding_template def update(self, instance, validated_data): - # Save vulnerability ids and pop them - if "vulnerability_id_template_set" in validated_data: - vulnerability_id_set = validated_data.pop( - "vulnerability_id_template_set", - ) - vulnerability_ids = [] - if vulnerability_id_set: - vulnerability_ids.extend(vulnerability_id["vulnerability_id"] for vulnerability_id in vulnerability_id_set) + # Handle vulnerability_ids if provided + if "vulnerability_ids" in self.initial_data: + vulnerability_ids = self.initial_data.get("vulnerability_ids", []) + if isinstance(vulnerability_ids, str): + vulnerability_ids = [vid.strip() for vid in vulnerability_ids.split("\n") if vid.strip()] + elif not isinstance(vulnerability_ids, list): + vulnerability_ids = [] save_vulnerability_ids_template(instance, vulnerability_ids) + # Handle endpoints if provided + if "endpoints" in self.initial_data: + endpoint_urls = self.initial_data.get("endpoints", []) + if isinstance(endpoint_urls, str): + endpoint_urls = [url.strip() for url in endpoint_urls.split("\n") if url.strip()] + elif not isinstance(endpoint_urls, list): + endpoint_urls = [] + save_endpoints_template(instance, endpoint_urls) + return super().update(instance, validated_data) diff --git a/dojo/api_v2/views.py b/dojo/api_v2/views.py index b321c35d558..01318a70bc5 100644 --- a/dojo/api_v2/views.py +++ b/dojo/api_v2/views.py @@ -4,6 +4,7 @@ from datetime import datetime from pathlib import Path +import pghistory import tagulous from crum import get_current_user from dateutil.relativedelta import relativedelta @@ -171,6 +172,7 @@ from dojo.risk_acceptance.queries import get_authorized_risk_acceptances from dojo.test.queries import get_authorized_test_imports, get_authorized_tests from dojo.tool_product.queries import get_authorized_tool_product_settings +from dojo.user.authentication import reset_token_for_user from dojo.user.utils import get_configuration_permissions_codenames from dojo.utils import ( async_delete, @@ -2410,6 +2412,19 @@ def destroy(self, request, *args, **kwargs): self.perform_destroy(instance) return Response(status=status.HTTP_204_NO_CONTENT) + @action( + detail=True, + methods=["post"], + url_path="reset_api_token", + permission_classes=(IsAuthenticated, permissions.IsSuperUserOrGlobalOwner), + filter_backends=[], + pagination_class=None, + ) + def reset_api_token(self, request, pk=None): + target_user = self.get_object() + reset_token_for_user(acting_user=request.user, target_user=target_user) + return Response(status=status.HTTP_204_NO_CONTENT) + # Authorization: superuser @extend_schema_view(**schema_with_prefetch()) @@ -2516,7 +2531,17 @@ def perform_create(self, serializer): if jira_project := (jira_helper.get_jira_project(jira_driver) if jira_driver else None): push_to_jira = push_to_jira or jira_project.push_all_issues + # Add pghistory context for audit trail (adds to existing middleware context). + # /api/vue is the Pro UI + source = "import_vue" if "/api/vue/" in self.request.path else "import_api" + pghistory.context( + source=source, + scan_type=serializer.validated_data.get("scan_type"), + ) serializer.save(push_to_jira=push_to_jira) + # Add test_id to pghistory context now that test is created + if test_id := serializer.data.get("test"): + pghistory.context(test_id=test_id) def get_queryset(self): return get_authorized_tests(Permissions.Import_Scan_Result) @@ -2664,7 +2689,22 @@ def perform_create(self, serializer): if jira_project := (jira_helper.get_jira_project(jira_driver) if jira_driver else None): push_to_jira = push_to_jira or jira_project.push_all_issues logger.debug("push_to_jira: %s", push_to_jira) + # Add pghistory context for audit trail (adds to existing middleware context) + # For reimport, test may already exist or be created during save + test_id = test.id if test else serializer.validated_data.get("test", {}) + if hasattr(test_id, "id"): + test_id = test_id.id + # /api/vue is the Pro UI + source = "reimport_vue" if "/api/vue/" in self.request.path else "reimport_api" + pghistory.context( + source=source, + test_id=test_id if isinstance(test_id, int) else None, + scan_type=serializer.validated_data.get("scan_type"), + ) serializer.save(push_to_jira=push_to_jira) + # Update test_id if it wasn't available before save + if test_id_from_response := serializer.data.get("test"): + pghistory.context(test_id=test_id_from_response) # Authorization: configuration @@ -2695,15 +2735,24 @@ class BurpRawRequestResponseViewSet( queryset = BurpRawRequestResponse.objects.none() filter_backends = (DjangoFilterBackend,) filterset_fields = ["finding"] + permission_classes = ( + IsAuthenticated, + permissions.UserHasFindingPermission, + ) def get_queryset(self): - results = BurpRawRequestResponse.objects.all() - empty_value = b"" - results = results.exclude( - burpRequestBase64__exact=empty_value, - burpResponseBase64__exact=empty_value, + return ( + BurpRawRequestResponse.objects.filter( + finding__in=get_authorized_findings( + Permissions.Finding_View, + ), + ) + .exclude( + burpRequestBase64__exact=b"", + burpResponseBase64__exact=b"", + ) + .order_by("id") ) - return results.order_by("id") # Authorization: superuser diff --git a/dojo/apps.py b/dojo/apps.py index f1b2769f760..06733c29771 100644 --- a/dojo/apps.py +++ b/dojo/apps.py @@ -91,7 +91,8 @@ def ready(self): # Configure audit system after all models are loaded # This must be done in ready() to avoid "Models aren't loaded yet" errors # Note: pghistory models are registered here (no database access), but trigger - # enabling is handled via management command to avoid database access warnings + # enabling is handled in the entrpoint script to avoid database access warnings + # during startup register_django_pghistory_models() configure_audit_system() diff --git a/dojo/auditlog.py b/dojo/auditlog.py index 1996fc1cca4..6747f4fa91b 100644 --- a/dojo/auditlog.py +++ b/dojo/auditlog.py @@ -1,23 +1,29 @@ """ Audit logging configuration for DefectDojo. -This module handles conditional registration of models with either django-auditlog -or django-pghistory based on the DD_AUDITLOG_TYPE setting. +This module handles registration of models with django-pghistory. +django-auditlog support has been removed. """ -import contextlib import logging +import os import sys +import time import pghistory from dateutil.relativedelta import relativedelta from django.apps import apps from django.conf import settings from django.core.management import call_command -from django.db import models +from django.db import connection, models from django.utils import timezone logger = logging.getLogger(__name__) +# FindingReviewers proxy model will be created lazily in register_django_pghistory_models() +# Cannot be defined at module level because Finding.reviewers.through requires +# Django's app registry to be ready (AppRegistryNotReady error) +# The function is called from DojoAppConfig.ready() which guarantees the registry is ready + def _flush_models_in_batches(models_to_flush, timestamp_field: str, retention_period: int, batch_size: int, max_batches: int, *, dry_run: bool = False) -> tuple[int, int, bool]: """ @@ -80,13 +86,6 @@ def _flush_models_in_batches(models_to_flush, timestamp_field: str, retention_pe return total_deleted, total_batches, reached_any_limit -def _flush_django_auditlog(retention_period: int, batch_size: int, max_batches: int, *, dry_run: bool = False) -> tuple[int, int, bool]: - # Import inside to avoid model import issues at startup - from auditlog.models import LogEntry # noqa: PLC0415 - - return _flush_models_in_batches([LogEntry], "timestamp", retention_period, batch_size, max_batches, dry_run=dry_run) - - def _iter_pghistory_event_models(): """Yield pghistory Event models registered under the dojo app.""" for model in apps.get_app_config("dojo").get_models(): @@ -107,8 +106,7 @@ def run_flush_auditlog(retention_period: int | None = None, *, dry_run: bool = False) -> tuple[int, int, bool]: """ - Deletes audit entries older than the configured retention from both - django-auditlog and django-pghistory log entries. + Deletes audit entries older than the configured retention from django-pghistory log entries. Returns a tuple of (deleted_total, batches_done, reached_limit). """ @@ -121,93 +119,13 @@ def run_flush_auditlog(retention_period: int | None = None, max_batches = max_batches if max_batches is not None else getattr(settings, "AUDITLOG_FLUSH_MAX_BATCHES", 100) phase = "DRY RUN" if dry_run else "Cleanup" - logger.info("Running %s for django-auditlog entries with %d Months retention across all backends", phase, retention_period) - d_deleted, d_batches, d_limit = _flush_django_auditlog(retention_period, batch_size, max_batches, dry_run=dry_run) logger.info("Running %s for django-pghistory entries with %d Months retention across all backends", phase, retention_period) p_deleted, p_batches, p_limit = _flush_pghistory_events(retention_period, batch_size, max_batches, dry_run=dry_run) - total_deleted = d_deleted + p_deleted - total_batches = d_batches + p_batches - reached_limit = bool(d_limit or p_limit) - verb = "would delete" if dry_run else "deleted" - logger.info("Audit flush summary: django-auditlog %s=%s batches=%s; pghistory %s=%s batches=%s; total_%s=%s total_batches=%s", - verb, d_deleted, d_batches, verb, p_deleted, p_batches, verb.replace(" ", "_"), total_deleted, total_batches) - - return total_deleted, total_batches, reached_limit - - -def enable_django_auditlog(): - """Enable django-auditlog by registering models.""" - # Import inside function to avoid AppRegistryNotReady errors - from auditlog.registry import auditlog # noqa: PLC0415 - - from dojo.models import ( # noqa: PLC0415 - Cred_User, - Dojo_User, - Endpoint, - Engagement, - Finding, - Finding_Group, - Finding_Template, - Notification_Webhooks, - Product, - Product_Type, - Risk_Acceptance, - Test, - ) - - logger.info("Enabling django-auditlog: Registering models") - auditlog.register(Dojo_User, exclude_fields=["password"]) - auditlog.register(Endpoint) - auditlog.register(Engagement) - auditlog.register(Finding, m2m_fields={"reviewers"}) - auditlog.register(Finding_Group) - auditlog.register(Product_Type) - auditlog.register(Product) - auditlog.register(Test) - auditlog.register(Risk_Acceptance) - auditlog.register(Finding_Template) - auditlog.register(Cred_User, exclude_fields=["password"]) - auditlog.register(Notification_Webhooks, exclude_fields=["header_name", "header_value"]) - logger.info("Successfully enabled django-auditlog") - - -def disable_django_auditlog(): - """Disable django-auditlog by unregistering models.""" - # Import inside function to avoid AppRegistryNotReady errors - from auditlog.registry import auditlog # noqa: PLC0415 - - from dojo.models import ( # noqa: PLC0415 - Cred_User, - Dojo_User, - Endpoint, - Engagement, - Finding, - Finding_Group, - Finding_Template, - Notification_Webhooks, - Product, - Product_Type, - Risk_Acceptance, - Test, - ) - - # Only log during actual application startup, not during shell commands - if "shell" not in sys.argv: - logger.info("Django-auditlog disabled - unregistering models") - - # Unregister all models from auditlog - models_to_unregister = [ - Dojo_User, Endpoint, Engagement, Finding, Finding_Group, - Product_Type, Product, Test, Risk_Acceptance, Finding_Template, - Cred_User, Notification_Webhooks, - ] + logger.info("Audit flush summary: pghistory %s=%s batches=%s", verb, p_deleted, p_batches) - for model in models_to_unregister: - with contextlib.suppress(Exception): - # Model might not be registered, ignore the error - auditlog.unregister(model) + return p_deleted, p_batches, bool(p_limit) def register_django_pghistory_models(): @@ -254,7 +172,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), exclude=["password"], # add some indexes manually so we don't have to define a customer phistory Event model with overridden fields. meta={ @@ -270,7 +188,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), meta={ "indexes": [ models.Index(fields=["pgh_created_at"]), @@ -284,7 +202,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), meta={ "indexes": [ models.Index(fields=["pgh_created_at"]), @@ -298,7 +216,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), meta={ "indexes": [ models.Index(fields=["pgh_created_at"]), @@ -312,7 +230,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), meta={ "indexes": [ models.Index(fields=["pgh_created_at"]), @@ -326,7 +244,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), meta={ "indexes": [ models.Index(fields=["pgh_created_at"]), @@ -340,7 +258,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), meta={ "indexes": [ models.Index(fields=["pgh_created_at"]), @@ -354,7 +272,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), meta={ "indexes": [ models.Index(fields=["pgh_created_at"]), @@ -368,7 +286,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), meta={ "indexes": [ models.Index(fields=["pgh_created_at"]), @@ -382,7 +300,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), meta={ "indexes": [ models.Index(fields=["pgh_created_at"]), @@ -396,7 +314,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), exclude=["password"], meta={ "indexes": [ @@ -411,7 +329,7 @@ def register_django_pghistory_models(): pghistory.InsertEvent(), pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)), pghistory.DeleteEvent(), - pghistory.ManualEvent(label="initial_import"), + pghistory.ManualEvent(label="initial_backfill"), exclude=["header_name", "header_value"], meta={ "indexes": [ @@ -422,35 +340,47 @@ def register_django_pghistory_models(): }, )(Notification_Webhooks) + # Track Finding.reviewers ManyToMany relationship + # Create a proxy model for the through table as per pghistory docs: + # https://django-pghistory.readthedocs.io/en/2.4.2/tutorial.html#tracking-many-to-many-events + # Note: For auto-generated through models, we don't specify obj_fk/obj_field + # as Django doesn't allow foreign keys to auto-generated through models + # + # We must create the proxy model here (not at module level) because: + # 1. Finding.reviewers.through requires Django's app registry to be ready + # 2. This function is called from DojoAppConfig.ready() which guarantees registry is ready + # 3. We check if it already exists to avoid re-registration warnings + # + # Note: This pattern is not explicitly documented in Django's official documentation. + # Django docs mention AppRegistryNotReady and AppConfig.ready() in general terms, but + # don't specifically cover proxy models for auto-generated ManyToMany through tables. + # This is a common pattern used by libraries like django-pghistory and is necessary + # because accessing Model.field.through at module import time triggers AppRegistryNotReady. + reviewers_through = Finding._meta.get_field("reviewers").remote_field.through + + class FindingReviewers(reviewers_through): + class Meta: + proxy = True + + pghistory.track( + pghistory.InsertEvent(), + pghistory.DeleteEvent(), + pghistory.ManualEvent(label="initial_backfill"), + meta={ + "db_table": "dojo_finding_reviewersevent", + "indexes": [ + models.Index(fields=["pgh_created_at"]), + models.Index(fields=["pgh_label"]), + models.Index(fields=["pgh_context_id"]), + ], + }, + )(FindingReviewers) + # Only log during actual application startup, not during shell commands if "shell" not in sys.argv: logger.info("Successfully registered models with django-pghistory") -def enable_django_pghistory(): - """Enable django-pghistory by enabling triggers.""" - logger.info("Enabling django-pghistory: Enabling triggers") - - # Enable pghistory triggers - try: - call_command("pgtrigger", "enable") - logger.info("Successfully enabled pghistory triggers") - except Exception as e: - logger.warning(f"Failed to enable pgtrigger triggers: {e}") - # Don't raise the exception as this shouldn't prevent Django from starting - - -def disable_django_pghistory(): - """Disable django-pghistory by disabling triggers.""" - logger.info("Disabling django-pghistory: Disabling triggers") - try: - call_command("pgtrigger", "disable") - logger.info("Successfully disabled pghistory triggers") - except Exception as e: - logger.warning(f"Failed to disable pgtrigger triggers: {e}") - # Don't raise the exception as this shouldn't prevent Django from starting - - def configure_pghistory_triggers(): """ Configure pghistory triggers based on audit settings. @@ -466,44 +396,399 @@ def configure_pghistory_triggers(): except Exception as e: logger.error(f"Failed to disable pghistory triggers: {e}") raise - elif settings.AUDITLOG_TYPE == "django-pghistory": + else: + # Only pghistory is supported now try: call_command("pgtrigger", "enable") logger.info("Successfully enabled pghistory triggers") except Exception as e: logger.error(f"Failed to enable pghistory triggers: {e}") raise - else: - try: - call_command("pgtrigger", "disable") - logger.info("Successfully disabled pghistory triggers") - except Exception as e: - logger.error(f"Failed to disable pghistory triggers: {e}") - raise def configure_audit_system(): """ Configure the audit system based on settings. - Note: This function only handles auditlog registration. pghistory model registration - is handled in apps.py, and trigger management should be done via the - configure_pghistory_triggers() function to avoid database access during initialization. + django-auditlog is no longer supported. Only django-pghistory is allowed. """ # Only log during actual application startup, not during shell commands log_enabled = "shell" not in sys.argv + # Fail if DD_AUDITLOG_TYPE is still configured (removed setting) + auditlog_type_env = os.environ.get("DD_AUDITLOG_TYPE") + if auditlog_type_env: + error_msg = ( + "DD_AUDITLOG_TYPE environment variable is no longer supported. " + "DefectDojo now exclusively uses django-pghistory for audit logging. " + "Please remove DD_AUDITLOG_TYPE from your environment configuration. " + "All new audit entries will be created using django-pghistory automatically." + ) + logger.error(error_msg) + raise ValueError(error_msg) + + # Fail if AUDITLOG_TYPE is manually set in settings files (removed setting) + if hasattr(settings, "AUDITLOG_TYPE"): + error_msg = ( + "AUDITLOG_TYPE setting is no longer supported. " + "DefectDojo now exclusively uses django-pghistory for audit logging. " + "Please remove AUDITLOG_TYPE from your settings file (settings.dist.py or local_settings.py). " + "All new audit entries will be created using django-pghistory automatically." + ) + logger.error(error_msg) + raise ValueError(error_msg) + if not settings.ENABLE_AUDITLOG: if log_enabled: logger.info("Audit logging disabled") - disable_django_auditlog() return - if settings.AUDITLOG_TYPE == "django-auditlog": - if log_enabled: - logger.info("Configuring audit system: django-auditlog enabled") - enable_django_auditlog() + if log_enabled: + logger.info("Audit logging configured: django-pghistory") + + +# Backfill functions for pghistory tables +def get_excluded_fields(model_name): + """Get the list of excluded fields for a specific model from pghistory configuration.""" + # Define excluded fields for each model (matching auditlog.py) + excluded_fields_map = { + "Dojo_User": ["password"], + "Product": ["updated"], # This is the key change + "Cred_User": ["password"], + "Notification_Webhooks": ["header_name", "header_value"], + } + return excluded_fields_map.get(model_name, []) + + +def get_table_names(model_name): + """Get the source table name and event table name for a model.""" + # Handle special cases for table naming + if model_name == "Dojo_User": + table_name = "dojo_dojo_user" + event_table_name = "dojo_dojo_userevent" + elif model_name == "Product_Type": + table_name = "dojo_product_type" + event_table_name = "dojo_product_typeevent" + elif model_name == "Finding_Group": + table_name = "dojo_finding_group" + event_table_name = "dojo_finding_groupevent" + elif model_name == "Risk_Acceptance": + table_name = "dojo_risk_acceptance" + event_table_name = "dojo_risk_acceptanceevent" + elif model_name == "Finding_Template": + table_name = "dojo_finding_template" + event_table_name = "dojo_finding_templateevent" + elif model_name == "Cred_User": + table_name = "dojo_cred_user" + event_table_name = "dojo_cred_userevent" + elif model_name == "Notification_Webhooks": + table_name = "dojo_notification_webhooks" + event_table_name = "dojo_notification_webhooksevent" + elif model_name == "FindingReviewers": + # M2M through table: Django creates dojo_finding_reviewers for Finding.reviewers + table_name = "dojo_finding_reviewers" + event_table_name = "dojo_finding_reviewersevent" else: - if log_enabled: - logger.info("django-auditlog disabled (pghistory or other audit type selected)") - disable_django_auditlog() + table_name = f"dojo_{model_name.lower()}" + event_table_name = f"dojo_{model_name.lower()}event" + return table_name, event_table_name + + +def check_tables_exist(table_name, event_table_name): + """Check if both source and event tables exist.""" + with connection.cursor() as cursor: + cursor.execute(""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = %s + ) + """, [table_name]) + table_exists = cursor.fetchone()[0] + + cursor.execute(""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = %s + ) + """, [event_table_name]) + event_table_exists = cursor.fetchone()[0] + + return table_exists, event_table_exists + + +def process_model_backfill( + model_name, + batch_size=10000, + *, + dry_run=False, + progress_callback=None, +): + """ + Process a single model's backfill using PostgreSQL COPY. + + Args: + model_name: Name of the model to backfill + batch_size: Number of records to process in each batch + dry_run: If True, only show what would be done without creating events + progress_callback: Optional callable that receives (message, style) tuples + for progress updates. If None, uses logger.info + + Returns: + tuple: (processed_count, records_per_second) + + """ + if progress_callback is None: + def progress_callback(msg, style=None): + logger.info(msg) + + try: + table_name, event_table_name = get_table_names(model_name) + + # Check if tables exist + table_exists, event_table_exists = check_tables_exist(table_name, event_table_name) + + if not table_exists: + progress_callback(f" Table {table_name} not found") + return 0, 0.0 + + if not event_table_exists: + progress_callback( + f" Event table {event_table_name} not found. " + f"Is {model_name} tracked by pghistory?", + "ERROR", + ) + return 0, 0.0 + + # Get total count using raw SQL + with connection.cursor() as cursor: + cursor.execute(f"SELECT COUNT(*) FROM {table_name}") + total_count = cursor.fetchone()[0] + + if total_count == 0: + progress_callback(f" No records found for {model_name}") + return 0, 0.0 + + progress_callback(f" Found {total_count:,} records") + + # Get excluded fields + excluded_fields = get_excluded_fields(model_name) + + # Check if records already have initial_backfill events using raw SQL + with connection.cursor() as cursor: + cursor.execute(f"SELECT COUNT(*) FROM {event_table_name} WHERE pgh_label = 'initial_backfill'") + existing_count = cursor.fetchone()[0] + + # Get records that need backfill using raw SQL + with connection.cursor() as cursor: + cursor.execute(f""" + SELECT COUNT(*) FROM {table_name} t + WHERE NOT EXISTS ( + SELECT 1 FROM {event_table_name} e + WHERE e.pgh_obj_id = t.id AND e.pgh_label = 'initial_backfill' + ) + """) + backfill_count = cursor.fetchone()[0] + + # Log the breakdown + progress_callback(f" Records with initial_backfill events: {existing_count:,}") + progress_callback(f" Records needing initial_backfill events: {backfill_count:,}") + + if backfill_count == 0: + progress_callback(f" ✓ All {total_count:,} records already have initial_backfill events", "SUCCESS") + return total_count, 0.0 + + if dry_run: + progress_callback(f" Would process {backfill_count:,} records using COPY...") + return backfill_count, 0.0 + + # Get event table columns using raw SQL (excluding auto-generated pgh_id) + with connection.cursor() as cursor: + cursor.execute(""" + SELECT column_name + FROM information_schema.columns + WHERE table_name = %s AND column_name != 'pgh_id' + ORDER BY ordinal_position + """, [event_table_name]) + event_columns = [row[0] for row in cursor.fetchall()] + + # Get all IDs that need backfill first + with connection.cursor() as cursor: + cursor.execute(f""" + SELECT t.id FROM {table_name} t + WHERE NOT EXISTS ( + SELECT 1 FROM {event_table_name} e + WHERE e.pgh_obj_id = t.id AND e.pgh_label = 'initial_backfill' + ) + ORDER BY t.id + """) + ids_to_process = [row[0] for row in cursor.fetchall()] + + if not ids_to_process: + progress_callback(" No records need backfill") + return 0, 0.0 + + # Process records in batches using raw SQL + processed = 0 + batch_start_time = time.time() + model_start_time = time.time() # Track model start time + + # Get column names for the source table + with connection.cursor() as cursor: + cursor.execute(""" + SELECT column_name + FROM information_schema.columns + WHERE table_name = %s + ORDER BY ordinal_position + """, [table_name]) + source_columns = [row[0] for row in cursor.fetchall()] + + # Filter out excluded fields from source columns + source_columns = [col for col in source_columns if col not in excluded_fields] + + # Find the index of the 'id' column for pgh_obj_id mapping + try: + id_column_index = source_columns.index("id") + except ValueError: + # If id is excluded (shouldn't happen), fall back to first column + id_column_index = 0 + progress_callback(" Warning: 'id' column not found in source columns, using first column", "WARNING") + + # Process in batches + consecutive_failures = 0 + max_failures = 3 + + for i in range(0, len(ids_to_process), batch_size): + batch_ids = ids_to_process[i:i + batch_size] + + # Log progress every 10 batches + if i > 0 and i % (batch_size * 10) == 0: + progress_callback(f" Processing batch starting at index {i:,}...") + + # Get batch of records using raw SQL with specific IDs + columns_str = ", ".join(source_columns) + placeholders = ", ".join(["%s"] * len(batch_ids)) + query = f""" + SELECT {columns_str} FROM {table_name} t + WHERE t.id IN ({placeholders}) + ORDER BY t.id + """ + + with connection.cursor() as cursor: + cursor.execute(query, batch_ids) + batch_rows = cursor.fetchall() + + if not batch_rows: + progress_callback(f" No records found for batch at index {i}") + continue + + # Use PostgreSQL COPY + try: + # Use PostgreSQL COPY with psycopg3 syntax + with connection.cursor() as cursor: + # Get the underlying raw cursor to bypass Django's wrapper + raw_cursor = cursor.cursor + # Use the copy method (psycopg3 syntax) + copy_sql = f"COPY {event_table_name} ({', '.join(event_columns)}) FROM STDIN WITH (FORMAT text, DELIMITER E'\\t')" + + # Use psycopg3 copy syntax as per documentation + # Prepare data as list of tuples for write_row() + records = [] + for row in batch_rows: + row_data = [] + + # Create a mapping of source columns to values + source_values = {} + for idx, value in enumerate(row): + field_name = source_columns[idx] + source_values[field_name] = value + + # Build row data in the order of event_columns + for col in event_columns: + if col == "pgh_created_at": + row_data.append(timezone.now()) + elif col == "pgh_label": + row_data.append("initial_backfill") + elif col == "pgh_obj_id": + # Use the id column index instead of assuming position + row_data.append(row[id_column_index] if row[id_column_index] is not None else None) + elif col == "pgh_context_id": + row_data.append(None) # Empty for backfilled events + elif col in source_values: + row_data.append(source_values[col]) + else: + row_data.append(None) # Default NULL value + + records.append(tuple(row_data)) + + # Use COPY with write_row() as per psycopg3 docs + with raw_cursor.copy(copy_sql) as copy: + for record in records: + copy.write_row(record) + progress_callback(" COPY operation completed using write_row") + + # Commit the transaction to persist the data + raw_cursor.connection.commit() + + # Debug: Check if data was inserted + raw_cursor.execute(f"SELECT COUNT(*) FROM {event_table_name} WHERE pgh_label = 'initial_backfill'") + count = raw_cursor.fetchone()[0] + progress_callback(f" Records in event table after batch: {count}") + + batch_processed = len(batch_rows) + processed += batch_processed + consecutive_failures = 0 # Reset failure counter on success + + # Calculate timing + batch_end_time = time.time() + batch_duration = batch_end_time - batch_start_time + batch_records_per_second = batch_processed / batch_duration if batch_duration > 0 else 0 + + # Log progress + progress = (processed / backfill_count) * 100 + progress_callback( + f" Processed {processed:,}/{backfill_count:,} records ({progress:.1f}%) - " + f"Last batch: {batch_duration:.2f}s ({batch_records_per_second:.1f} records/sec)", + ) + + batch_start_time = time.time() # Reset for next batch + + except Exception as e: + consecutive_failures += 1 + logger.error(f"Bulk insert failed for {model_name} batch: {e}") + progress_callback(f" Bulk insert failed: {e}", "ERROR") + # Log more details about the error + progress_callback(f" Processed {processed:,} records before failure") + + if consecutive_failures >= max_failures: + progress_callback(f" Too many consecutive failures ({consecutive_failures}), stopping processing", "ERROR") + break + + # Continue with next batch instead of breaking + continue + + # Calculate total timing + model_end_time = time.time() + total_duration = model_end_time - model_start_time + records_per_second = processed / total_duration if total_duration > 0 else 0 + + progress_callback( + f" ✓ Completed {model_name}: {processed:,} records in {total_duration:.2f}s " + f"({records_per_second:.1f} records/sec)", + "SUCCESS", + ) + except Exception as e: + progress_callback(f" ✗ Failed to process {model_name}: {e}", "ERROR") + logger.exception(f"Error processing {model_name}") + return 0, 0.0 + else: + return processed, records_per_second + + +def get_tracked_models(): + """Get the list of models tracked by pghistory.""" + return [ + "Dojo_User", "Endpoint", "Engagement", "Finding", "Finding_Group", + "Product_Type", "Product", "Test", "Risk_Acceptance", + "Finding_Template", "Cred_User", "Notification_Webhooks", + "FindingReviewers", # M2M through table for Finding.reviewers + ] diff --git a/dojo/authorization/authorization.py b/dojo/authorization/authorization.py index 4110b965db1..b410bb3a95d 100644 --- a/dojo/authorization/authorization.py +++ b/dojo/authorization/authorization.py @@ -40,6 +40,35 @@ def user_has_configuration_permission(user, permission): return user.has_perm(permission) +def user_is_superuser_or_global_owner(user): + """ + Returns True if the user is a superuser or has a global role (directly or + via group membership) whose Role.is_owner is True. + """ + if not user or getattr(user, "is_anonymous", False): + return False + + if user.is_superuser: + return True + + if ( + hasattr(user, "global_role") + and user.global_role.role is not None + and user.global_role.role.is_owner + ): + return True + + for group in get_groups(user): + if ( + hasattr(group, "global_role") + and group.global_role.role is not None + and group.global_role.role.is_owner + ): + return True + + return False + + def user_has_permission(user, obj, permission): if user.is_anonymous: return False diff --git a/dojo/celery.py b/dojo/celery.py index 5f2935b4460..ead4a8813a8 100644 --- a/dojo/celery.py +++ b/dojo/celery.py @@ -2,7 +2,7 @@ import os from logging.config import dictConfig -from celery import Celery +from celery import Celery, Task from celery.signals import setup_logging from django.conf import settings @@ -11,7 +11,31 @@ # set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dojo.settings.settings") -app = Celery("dojo") + +class PgHistoryTask(Task): + + """ + Custom Celery base task that automatically applies pghistory context. + + When a task is dispatched via dojo_async_task, the current pghistory + context is captured and passed in kwargs as "_pgh_context". This base + class extracts that context and applies it before running the task, + ensuring all database events share the same context as the original + request. + """ + + def __call__(self, *args, **kwargs): + # Import here to avoid circular imports during Celery startup + from dojo.pghistory_utils import get_pghistory_context_manager # noqa: PLC0415 + + # Extract context from kwargs (won't be passed to task function) + pgh_context = kwargs.pop("_pgh_context", None) + + with get_pghistory_context_manager(pgh_context): + return super().__call__(*args, **kwargs) + + +app = Celery("dojo", task_cls=PgHistoryTask) # Using a string here means the worker will not have to # pickle the object when using Windows. diff --git a/dojo/context_processors.py b/dojo/context_processors.py index 409851e2458..6f6bef626d6 100644 --- a/dojo/context_processors.py +++ b/dojo/context_processors.py @@ -3,6 +3,7 @@ # import the settings file from django.conf import settings +from django.contrib import messages from dojo.labels import get_labels from dojo.models import Alerts, System_Settings, UserAnnouncement @@ -39,7 +40,26 @@ def globalize_vars(request): def bind_system_settings(request): - return {"system_settings": System_Settings.objects.get()} + """Load system settings and display warning if there's a database error.""" + try: + system_settings = System_Settings.objects.get() + # Check if there was an error stored on the request (from middleware) + if hasattr(request, "system_settings_error"): + error_msg = request.system_settings_error + messages.add_message( + request, + messages.WARNING, + f"Warning: Unable to load system settings from database: {error_msg}. " + "Default values are being used. Please check your database configuration and run migrations if needed.", + extra_tags="alert-warning", + ) + # Clear after adding message + delattr(request, "system_settings_error") + except Exception: + # If we can't get settings, return empty dict (will cause errors elsewhere, but that's expected) + return {} + + return {"system_settings": system_settings} def bind_alert_count(request): diff --git a/dojo/db_migrations/0249_findingreviewers_findingreviewersevent_and_more.py b/dojo/db_migrations/0249_findingreviewers_findingreviewersevent_and_more.py new file mode 100644 index 00000000000..c338138055c --- /dev/null +++ b/dojo/db_migrations/0249_findingreviewers_findingreviewersevent_and_more.py @@ -0,0 +1,64 @@ +# Generated by Django 5.1.13 on 2025-11-01 17:04 + +import django.db.models.deletion +import pgtrigger.compiler +import pgtrigger.migrations +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("dojo", "0248_alter_general_survey_expiration") + ] + + operations = [ + migrations.CreateModel( + name="FindingReviewers", + fields=[ + ], + options={ + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("dojo.finding_reviewers",), + ), + migrations.CreateModel( + name="FindingReviewersEvent", + fields=[ + ("pgh_id", models.AutoField(primary_key=True, serialize=False)), + ("pgh_created_at", models.DateTimeField(auto_now_add=True)), + ("pgh_label", models.TextField(help_text="The event label.")), + ("id", models.IntegerField()), + ("dojo_user", models.ForeignKey(db_constraint=False, db_index=False, db_tablespace="", on_delete=django.db.models.deletion.DO_NOTHING, related_name="+", related_query_name="+", to="dojo.dojo_user")), + ("finding", models.ForeignKey(db_constraint=False, db_index=False, db_tablespace="", on_delete=django.db.models.deletion.DO_NOTHING, related_name="+", related_query_name="+", to="dojo.finding")), + ("pgh_context", models.ForeignKey(db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name="+", to="pghistory.context")), + ("pgh_obj", models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, related_name="events", to="dojo.findingreviewers")), + ], + options={ + "abstract": False, + "db_table": "dojo_finding_reviewersevent", + }, + ), + migrations.AddIndex( + model_name="findingreviewersevent", + index=models.Index(fields=["pgh_created_at"], name="dojo_findin_pgh_cre_d5e5b4_idx"), + ), + migrations.AddIndex( + model_name="findingreviewersevent", + index=models.Index(fields=["pgh_label"], name="dojo_findin_pgh_lab_5517f9_idx"), + ), + migrations.AddIndex( + model_name="findingreviewersevent", + index=models.Index(fields=["pgh_context_id"], name="dojo_findin_pgh_con_06229b_idx"), + ), + pgtrigger.migrations.AddTrigger( + model_name="findingreviewers", + trigger=pgtrigger.compiler.Trigger(name="insert_insert", sql=pgtrigger.compiler.UpsertTriggerSql(func='INSERT INTO "dojo_finding_reviewersevent" ("dojo_user_id", "finding_id", "id", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id") VALUES (NEW."dojo_user_id", NEW."finding_id", NEW."id", _pgh_attach_context(), NOW(), \'insert\', NEW."id"); RETURN NULL;', hash="5c1fd440159e49c929122cbb590f96983a1c934e", operation="INSERT", pgid="pgtrigger_insert_insert_0808c", table="dojo_finding_reviewers", when="AFTER")), + ), + pgtrigger.migrations.AddTrigger( + model_name="findingreviewers", + trigger=pgtrigger.compiler.Trigger(name="delete_delete", sql=pgtrigger.compiler.UpsertTriggerSql(func='INSERT INTO "dojo_finding_reviewersevent" ("dojo_user_id", "finding_id", "id", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id") VALUES (OLD."dojo_user_id", OLD."finding_id", OLD."id", _pgh_attach_context(), NOW(), \'delete\', OLD."id"); RETURN NULL;', hash="23a4e01eaea469f708679392a6a92a6e16b21181", operation="DELETE", pgid="pgtrigger_delete_delete_40083", table="dojo_finding_reviewers", when="AFTER")), + ), + ] diff --git a/dojo/db_migrations/0250_pghistory_backfill.py b/dojo/db_migrations/0250_pghistory_backfill.py new file mode 100644 index 00000000000..5d09d77c0e2 --- /dev/null +++ b/dojo/db_migrations/0250_pghistory_backfill.py @@ -0,0 +1,85 @@ +# Generated manually for pghistory initial backfill + +import logging + +from django.conf import settings +from django.db import migrations + +from dojo.auditlog import ( + get_tracked_models, + process_model_backfill, +) + +logger = logging.getLogger(__name__) + + +def backfill_pghistory_tables(apps, schema_editor): + """ + Backfill pghistory tables with initial snapshots of existing records. + + This migration is fail-safe: if it fails for some reason, it will continue + where it left off on the next run, as it only processes records that don't + already have initial_backfill events. + """ + # Skip if auditlog is not enabled + if not settings.ENABLE_AUDITLOG: + logger.info("pghistory is not enabled. Skipping backfill.") + return + + # Check if we can use COPY (PostgreSQL only) + if settings.DATABASES["default"]["ENGINE"] != "django.db.backends.postgresql": + logger.warning( + "COPY operations only available with PostgreSQL. " + "Skipping backfill. Use the pghistory_backfill command instead.", + ) + return + + # Progress callback for migration logging + def progress_callback(msg, style=None): + """Progress callback that logs to Django's logger.""" + if style == "ERROR": + logger.error(msg) + elif style == "WARNING": + logger.warning(msg) + elif style == "SUCCESS": + logger.info(msg) + else: + logger.info(msg) + + # Get all tracked models + tracked_models = get_tracked_models() + + logger.info(f"Starting pghistory backfill for {len(tracked_models)} model(s)...") + + total_processed = 0 + for model_name in tracked_models: + logger.info(f"Processing {model_name}...") + try: + processed, _ = process_model_backfill( + model_name=model_name, + batch_size=10000, + dry_run=False, + progress_callback=progress_callback, + ) + total_processed += processed + except Exception as e: + logger.error(f"Failed to backfill {model_name}: {e}", exc_info=True) + # Continue with other models even if one fails + continue + + logger.info(f"Pghistory backfill complete: Processed {total_processed:,} records") + + +class Migration(migrations.Migration): + + dependencies = [ + ("dojo", "0249_findingreviewers_findingreviewersevent_and_more"), + ] + + operations = [ + migrations.RunPython( + backfill_pghistory_tables, + reverse_code=migrations.RunPython.noop, + ), + ] + diff --git a/dojo/db_migrations/0251_usercontactinfo_reset_timestamps.py b/dojo/db_migrations/0251_usercontactinfo_reset_timestamps.py new file mode 100644 index 00000000000..88d071182f3 --- /dev/null +++ b/dojo/db_migrations/0251_usercontactinfo_reset_timestamps.py @@ -0,0 +1,33 @@ +# Generated by Django 3.1.13 on 2025-12-12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("dojo", "0250_pghistory_backfill"), + ] + + operations = [ + migrations.AddField( + model_name="usercontactinfo", + name="token_last_reset", + field=models.DateTimeField( + blank=True, + help_text="Timestamp of the most recent API token reset for this user.", + null=True, + ), + ), + migrations.AddField( + model_name="usercontactinfo", + name="password_last_reset", + field=models.DateTimeField( + blank=True, + help_text="Timestamp of the most recent password reset for this user.", + null=True, + ), + ), + ] + + diff --git a/dojo/db_migrations/0252_finding_template_changes.py b/dojo/db_migrations/0252_finding_template_changes.py new file mode 100644 index 00000000000..4fd2ac3a304 --- /dev/null +++ b/dojo/db_migrations/0252_finding_template_changes.py @@ -0,0 +1,200 @@ +# Generated by Django 5.2.9 on 2025-12-21 07:29 + +import dojo.validators +import pgtrigger.compiler +import pgtrigger.migrations +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0251_usercontactinfo_reset_timestamps'), + ] + + operations = [ + pgtrigger.migrations.RemoveTrigger( + model_name='finding_template', + name='insert_insert', + ), + pgtrigger.migrations.RemoveTrigger( + model_name='finding_template', + name='update_update', + ), + pgtrigger.migrations.RemoveTrigger( + model_name='finding_template', + name='delete_delete', + ), + migrations.RemoveField( + model_name='system_settings', + name='enable_template_match', + ), + migrations.RemoveField( + model_name='finding_template', + name='template_match', + ), + migrations.RemoveField( + model_name='finding_template', + name='template_match_title', + ), + migrations.RemoveField( + model_name='finding_templateevent', + name='template_match', + ), + migrations.RemoveField( + model_name='finding_templateevent', + name='template_match_title', + ), + migrations.AddField( + model_name='finding_template', + name='cvssv4', + field=models.TextField(help_text='Common Vulnerability Scoring System version 4 (CVSS4) score associated with this finding.', max_length=255, null=True, validators=[dojo.validators.cvss4_validator], verbose_name='CVSS4 vector'), + ), + migrations.AddField( + model_name='finding_template', + name='component_name', + field=models.CharField(blank=True, help_text='Affected component name (e.g., library name)', max_length=500, null=True), + ), + migrations.AddField( + model_name='finding_template', + name='component_version', + field=models.CharField(blank=True, help_text='Affected component version', max_length=100, null=True), + ), + migrations.AddField( + model_name='finding_template', + name='cvssv3_score', + field=models.FloatField(blank=True, help_text='CVSSv3 score', null=True), + ), + migrations.AddField( + model_name='finding_template', + name='cvssv4_score', + field=models.FloatField(blank=True, help_text='CVSSv4 score', null=True), + ), + migrations.AddField( + model_name='finding_template', + name='effort_for_fixing', + field=models.CharField(blank=True, help_text='Effort estimate for fixing (e.g., Low/Medium/High)', max_length=99, null=True), + ), + migrations.AddField( + model_name='finding_template', + name='fix_available', + field=models.BooleanField(blank=True, help_text='Indicates if a fix is available for this vulnerability type', null=True), + ), + migrations.AddField( + model_name='finding_template', + name='fix_version', + field=models.CharField(blank=True, help_text='Version where fix is available', max_length=100, null=True), + ), + migrations.AddField( + model_name='finding_template', + name='notes', + field=models.TextField(blank=True, help_text='Note content to add when applying this template', null=True), + ), + migrations.AddField( + model_name='finding_template', + name='planned_remediation_version', + field=models.CharField(blank=True, help_text='Target version for remediation', max_length=99, null=True), + ), + migrations.AddField( + model_name='finding_template', + name='severity_justification', + field=models.TextField(blank=True, help_text='Explanation of why this severity level is appropriate', null=True), + ), + migrations.AddField( + model_name='finding_template', + name='steps_to_reproduce', + field=models.TextField(blank=True, help_text='Standard reproduction steps for this vulnerability type', null=True), + ), + migrations.AddField( + model_name='finding_template', + name='vulnerability_ids_text', + field=models.TextField(blank=True, help_text='Vulnerability IDs (one per line)', null=True), + ), + migrations.AddField( + model_name='finding_template', + name='endpoints_text', + field=models.TextField(blank=True, help_text='Endpoint URLs (one per line)', null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='component_name', + field=models.CharField(blank=True, help_text='Affected component name (e.g., library name)', max_length=500, null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='component_version', + field=models.CharField(blank=True, help_text='Affected component version', max_length=100, null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='cvssv3_score', + field=models.FloatField(blank=True, help_text='CVSSv3 score', null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='cvssv4', + field=models.TextField(help_text='Common Vulnerability Scoring System version 4 (CVSS4) score associated with this finding.', max_length=255, null=True, validators=[dojo.validators.cvss4_validator], verbose_name='CVSS4 vector'), + ), + migrations.AddField( + model_name='finding_templateevent', + name='cvssv4_score', + field=models.FloatField(blank=True, help_text='CVSSv4 score', null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='effort_for_fixing', + field=models.CharField(blank=True, help_text='Effort estimate for fixing (e.g., Low/Medium/High)', max_length=99, null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='fix_available', + field=models.BooleanField(blank=True, help_text='Indicates if a fix is available for this vulnerability type', null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='fix_version', + field=models.CharField(blank=True, help_text='Version where fix is available', max_length=100, null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='notes', + field=models.TextField(blank=True, help_text='Note content to add when applying this template', null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='planned_remediation_version', + field=models.CharField(blank=True, help_text='Target version for remediation', max_length=99, null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='severity_justification', + field=models.TextField(blank=True, help_text='Explanation of why this severity level is appropriate', null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='steps_to_reproduce', + field=models.TextField(blank=True, help_text='Standard reproduction steps for this vulnerability type', null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='vulnerability_ids_text', + field=models.TextField(blank=True, help_text='Vulnerability IDs (one per line)', null=True), + ), + migrations.AddField( + model_name='finding_templateevent', + name='endpoints_text', + field=models.TextField(blank=True, help_text='Endpoint URLs (one per line)', null=True), + ), + pgtrigger.migrations.AddTrigger( + model_name='finding_template', + trigger=pgtrigger.compiler.Trigger(name='insert_insert', sql=pgtrigger.compiler.UpsertTriggerSql(func='INSERT INTO "dojo_finding_templateevent" ("component_name", "component_version", "cve", "cvssv3", "cvssv3_score", "cvssv4", "cvssv4_score", "cwe", "description", "effort_for_fixing", "endpoints_text", "fix_available", "fix_version", "id", "impact", "last_used", "mitigation", "notes", "numerical_severity", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "planned_remediation_version", "refs", "severity", "severity_justification", "steps_to_reproduce", "title", "vulnerability_ids_text") VALUES (NEW."component_name", NEW."component_version", NEW."cve", NEW."cvssv3", NEW."cvssv3_score", NEW."cvssv4", NEW."cvssv4_score", NEW."cwe", NEW."description", NEW."effort_for_fixing", NEW."endpoints_text", NEW."fix_available", NEW."fix_version", NEW."id", NEW."impact", NEW."last_used", NEW."mitigation", NEW."notes", NEW."numerical_severity", _pgh_attach_context(), NOW(), \'insert\', NEW."id", NEW."planned_remediation_version", NEW."refs", NEW."severity", NEW."severity_justification", NEW."steps_to_reproduce", NEW."title", NEW."vulnerability_ids_text"); RETURN NULL;', hash='602d9e872906719a1c95d671a0c9e4ffe5b1b7ec', operation='INSERT', pgid='pgtrigger_insert_insert_59092', table='dojo_finding_template', when='AFTER')), + ), + pgtrigger.migrations.AddTrigger( + model_name='finding_template', + trigger=pgtrigger.compiler.Trigger(name='update_update', sql=pgtrigger.compiler.UpsertTriggerSql(condition='WHEN (OLD.* IS DISTINCT FROM NEW.*)', func='INSERT INTO "dojo_finding_templateevent" ("component_name", "component_version", "cve", "cvssv3", "cvssv3_score", "cvssv4", "cvssv4_score", "cwe", "description", "effort_for_fixing", "endpoints_text", "fix_available", "fix_version", "id", "impact", "last_used", "mitigation", "notes", "numerical_severity", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "planned_remediation_version", "refs", "severity", "severity_justification", "steps_to_reproduce", "title", "vulnerability_ids_text") VALUES (NEW."component_name", NEW."component_version", NEW."cve", NEW."cvssv3", NEW."cvssv3_score", NEW."cvssv4", NEW."cvssv4_score", NEW."cwe", NEW."description", NEW."effort_for_fixing", NEW."endpoints_text", NEW."fix_available", NEW."fix_version", NEW."id", NEW."impact", NEW."last_used", NEW."mitigation", NEW."notes", NEW."numerical_severity", _pgh_attach_context(), NOW(), \'update\', NEW."id", NEW."planned_remediation_version", NEW."refs", NEW."severity", NEW."severity_justification", NEW."steps_to_reproduce", NEW."title", NEW."vulnerability_ids_text"); RETURN NULL;', hash='644c1afec5497d6e86c4c1c861801819b7363d61', operation='UPDATE', pgid='pgtrigger_update_update_43036', table='dojo_finding_template', when='AFTER')), + ), + pgtrigger.migrations.AddTrigger( + model_name='finding_template', + trigger=pgtrigger.compiler.Trigger(name='delete_delete', sql=pgtrigger.compiler.UpsertTriggerSql(func='INSERT INTO "dojo_finding_templateevent" ("component_name", "component_version", "cve", "cvssv3", "cvssv3_score", "cvssv4", "cvssv4_score", "cwe", "description", "effort_for_fixing", "endpoints_text", "fix_available", "fix_version", "id", "impact", "last_used", "mitigation", "notes", "numerical_severity", "pgh_context_id", "pgh_created_at", "pgh_label", "pgh_obj_id", "planned_remediation_version", "refs", "severity", "severity_justification", "steps_to_reproduce", "title", "vulnerability_ids_text") VALUES (OLD."component_name", OLD."component_version", OLD."cve", OLD."cvssv3", OLD."cvssv3_score", OLD."cvssv4", OLD."cvssv4_score", OLD."cwe", OLD."description", OLD."effort_for_fixing", OLD."endpoints_text", OLD."fix_available", OLD."fix_version", OLD."id", OLD."impact", OLD."last_used", OLD."mitigation", OLD."notes", OLD."numerical_severity", _pgh_attach_context(), NOW(), \'delete\', OLD."id", OLD."planned_remediation_version", OLD."refs", OLD."severity", OLD."severity_justification", OLD."steps_to_reproduce", OLD."title", OLD."vulnerability_ids_text"); RETURN NULL;', hash='925844fc74c390b9d4fc446e0d3f44f556817fd4', operation='DELETE', pgid='pgtrigger_delete_delete_3f3a6', table='dojo_finding_template', when='AFTER')), + ) + ] diff --git a/dojo/db_migrations/0253_migrate_vulnerability_ids_to_textfield.py b/dojo/db_migrations/0253_migrate_vulnerability_ids_to_textfield.py new file mode 100644 index 00000000000..d69ddb8a857 --- /dev/null +++ b/dojo/db_migrations/0253_migrate_vulnerability_ids_to_textfield.py @@ -0,0 +1,105 @@ +# Generated migration for migrating Vulnerability_Id_Template to vulnerability_ids_text + +from django.db import migrations + + +def migrate_vulnerability_ids_to_textfield(apps, schema_editor): + """ + Migrate Vulnerability_Id_Template records to vulnerability_ids_text TextField. + Converts all vulnerability IDs for each Finding_Template into a newline-separated string. + """ + Finding_Template = apps.get_model('dojo', 'Finding_Template') + Vulnerability_Id_Template = apps.get_model('dojo', 'Vulnerability_Id_Template') + + # Process each Finding_Template + for template in Finding_Template.objects.all(): + # Get all vulnerability IDs for this template + vuln_ids = Vulnerability_Id_Template.objects.filter( + finding_template=template + ).values_list('vulnerability_id', flat=True).order_by('id') + + # Convert to list and remove duplicates while preserving order + vulnerability_ids = list(dict.fromkeys([vid.strip() for vid in vuln_ids if vid.strip()])) + + # Save as newline-separated string + if vulnerability_ids: + template.vulnerability_ids_text = '\n'.join(vulnerability_ids) + # Also update CVE field for backward compatibility (first ID) + if not template.cve: + template.cve = vulnerability_ids[0] + else: + # If no vulnerability IDs, check if CVE field has a value + if template.cve: + template.vulnerability_ids_text = template.cve + else: + template.vulnerability_ids_text = None + + template.save(update_fields=['vulnerability_ids_text', 'cve']) + + +def reverse_migrate_vulnerability_ids_to_textfield(apps, schema_editor): + """ + Reverse migration: populate Vulnerability_Id_Template from vulnerability_ids_text. + Note: This will recreate Vulnerability_Id_Template records if they were deleted. + """ + Finding_Template = apps.get_model('dojo', 'Finding_Template') + Vulnerability_Id_Template = apps.get_model('dojo', 'Vulnerability_Id_Template') + + # Process each Finding_Template + for template in Finding_Template.objects.all(): + # Parse vulnerability_ids_text + vulnerability_ids = [] + if template.vulnerability_ids_text: + vulnerability_ids = [ + vid.strip() + for vid in template.vulnerability_ids_text.split('\n') + if vid.strip() + ] + + # Also check CVE field for backward compatibility + if not vulnerability_ids and template.cve: + vulnerability_ids = [template.cve] + + # Remove duplicates while preserving order + vulnerability_ids = list(dict.fromkeys(vulnerability_ids)) + + # Delete existing Vulnerability_Id_Template records for this template + Vulnerability_Id_Template.objects.filter(finding_template=template).delete() + + # Create new Vulnerability_Id_Template records + for vuln_id in vulnerability_ids: + Vulnerability_Id_Template.objects.create( + finding_template=template, + vulnerability_id=vuln_id + ) + + +def delete_old_vulnerability_id_templates(apps, schema_editor): + """ + Delete all Vulnerability_Id_Template records after migration is complete. + This is a one-way operation and cannot be reversed. + """ + Vulnerability_Id_Template = apps.get_model('dojo', 'Vulnerability_Id_Template') + Vulnerability_Id_Template.objects.all().delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0252_finding_template_changes'), + ] + + operations = [ + # Step 1: Migrate data from Vulnerability_Id_Template to vulnerability_ids_text + migrations.RunPython( + migrate_vulnerability_ids_to_textfield, + reverse_code=reverse_migrate_vulnerability_ids_to_textfield, + ), + # Step 2: Delete old Vulnerability_Id_Template records + # Note: This is irreversible - if you need to rollback, you'll need to restore from backup + migrations.RunPython( + delete_old_vulnerability_id_templates, + reverse_code=migrations.RunPython.noop, + ), + ] + diff --git a/dojo/db_migrations/0254_remove_vulnerability_id_template_model.py b/dojo/db_migrations/0254_remove_vulnerability_id_template_model.py new file mode 100644 index 00000000000..5c671726947 --- /dev/null +++ b/dojo/db_migrations/0254_remove_vulnerability_id_template_model.py @@ -0,0 +1,17 @@ +# Generated migration to remove Vulnerability_Id_Template model + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0253_migrate_vulnerability_ids_to_textfield'), + ] + + operations = [ + migrations.DeleteModel( + name='Vulnerability_Id_Template', + ), + ] + diff --git a/dojo/decorators.py b/dojo/decorators.py index bba9efe234c..91f6934b719 100644 --- a/dojo/decorators.py +++ b/dojo/decorators.py @@ -83,10 +83,17 @@ def dojo_async_task(func=None, *, signature=False): def decorator(func): @wraps(func) def __wrapper__(*args, **kwargs): + from dojo.pghistory_utils import get_serializable_pghistory_context # noqa: PLC0415 circular import from dojo.utils import get_current_user # noqa: PLC0415 circular import + user = get_current_user() kwargs["async_user"] = user + # Capture pghistory context to pass to Celery worker + # The PgHistoryTask base class will apply this context in the worker + if pgh_context := get_serializable_pghistory_context(): + kwargs["_pgh_context"] = pgh_context + dojo_async_task_counter.incr( func.__name__, args=args, diff --git a/dojo/endpoint/signals.py b/dojo/endpoint/signals.py index 50251c5a80a..aebc348c003 100644 --- a/dojo/endpoint/signals.py +++ b/dojo/endpoint/signals.py @@ -1,9 +1,7 @@ import contextlib -from auditlog.models import LogEntry from django.conf import settings from django.contrib.auth import get_user_model -from django.contrib.contenttypes.models import ContentType from django.db.models.signals import post_delete from django.dispatch import receiver from django.urls import reverse @@ -23,7 +21,7 @@ def endpoint_post_delete(sender, instance, using, origin, **kwargs): user = None if settings.ENABLE_AUDITLOG: - # First try to find deletion author in pghistory events + # Find deletion author in pghistory events # Look for delete events for this specific endpoint instance pghistory_delete_events = DojoEvents.objects.filter( pgh_obj_model="dojo.Endpoint", @@ -39,15 +37,6 @@ def endpoint_post_delete(sender, instance, using, origin, **kwargs): with contextlib.suppress(User.DoesNotExist): user = User.objects.get(id=latest_delete.user) - # Fall back to django-auditlog if no user found in pghistory - if not user: - if le := LogEntry.objects.filter( - action=LogEntry.Action.DELETE, - content_type=ContentType.objects.get(app_label="dojo", model="endpoint"), - object_id=instance.id, - ).order_by("-id").first(): - user = le.actor - # Update description with user if found if user: description = _('The endpoint "%(name)s" was deleted by %(user)s') % { diff --git a/dojo/engagement/signals.py b/dojo/engagement/signals.py index 144094a3264..0d6b8916dd2 100644 --- a/dojo/engagement/signals.py +++ b/dojo/engagement/signals.py @@ -1,9 +1,7 @@ import contextlib -from auditlog.models import LogEntry from django.conf import settings from django.contrib.auth import get_user_model -from django.contrib.contenttypes.models import ContentType from django.db.models.signals import post_delete, post_save, pre_delete, pre_save from django.dispatch import receiver from django.urls import reverse @@ -50,7 +48,7 @@ def engagement_post_delete(sender, instance, using, origin, **kwargs): user = None if settings.ENABLE_AUDITLOG: - # First try to find deletion author in pghistory events + # Find deletion author in pghistory events # Look for delete events for this specific engagement instance pghistory_delete_events = DojoEvents.objects.filter( pgh_obj_model="dojo.Engagement", @@ -66,15 +64,6 @@ def engagement_post_delete(sender, instance, using, origin, **kwargs): with contextlib.suppress(User.DoesNotExist): user = User.objects.get(id=latest_delete.user) - # Fall back to django-auditlog if no user found in pghistory - if not user: - if le := LogEntry.objects.filter( - action=LogEntry.Action.DELETE, - content_type=ContentType.objects.get(app_label="dojo", model="engagement"), - object_id=instance.id, - ).order_by("-id").first(): - user = le.actor - # Update description with user if found if user: description = _('The engagement "%(name)s" was deleted by %(user)s') % { diff --git a/dojo/engagement/views.py b/dojo/engagement/views.py index a726f514421..9ecdabfdd9b 100644 --- a/dojo/engagement/views.py +++ b/dojo/engagement/views.py @@ -10,6 +10,7 @@ from tempfile import NamedTemporaryFile from time import strftime +import pghistory from django.conf import settings from django.contrib import messages from django.contrib.admin.utils import NestedObjects @@ -1138,10 +1139,18 @@ def post( if form_error := self.process_form(request, context.get("form"), context): add_error_message_to_response(form_error) return self.failure_redirect(request, context) + # Add pghistory context for audit trail (adds to existing middleware context) + pghistory.context( + source="import", + scan_type=context.get("scan_type"), + ) # Kick off the import process if import_error := self.import_findings(context): add_error_message_to_response(import_error) return self.failure_redirect(request, context) + # Add test_id to pghistory context now that test is created + if test := context.get("test"): + pghistory.context(test_id=test.id) # Process the credential form if form_error := self.process_credentials_form(request, context.get("cred_form"), context): add_error_message_to_response(form_error) diff --git a/dojo/filters.py b/dojo/filters.py index 8e77ddac3aa..4ae5224dab6 100644 --- a/dojo/filters.py +++ b/dojo/filters.py @@ -93,7 +93,7 @@ from dojo.risk_acceptance.queries import get_authorized_risk_acceptances from dojo.test.queries import get_authorized_tests from dojo.user.queries import get_authorized_users -from dojo.utils import get_system_setting, is_finding_groups_enabled, truncate_timezone_aware +from dojo.utils import get_system_setting, get_visible_scan_types, is_finding_groups_enabled, truncate_timezone_aware logger = logging.getLogger(__name__) @@ -2075,6 +2075,9 @@ def __init__(self, *args, **kwargs): # Don't show the product filter on the product finding view self.set_related_object_fields(*args, **kwargs) + if "test__test_type" in self.form.fields: + self.form.fields["test__test_type"].queryset = get_visible_scan_types() + def set_related_object_fields(self, *args: list, **kwargs: dict): finding_group_query = Finding_Group.objects.all() if self.pid is not None: @@ -3604,7 +3607,7 @@ class PgHistoryFilter(DojoFilter): ("insert", "Insert"), ("update", "Update"), ("delete", "Delete"), - ("initial_import", "Initial Import"), + ("initial_backfill", "Initial Backfill"), ], ) diff --git a/dojo/finding/deduplication.py b/dojo/finding/deduplication.py index 14e4d33477c..fea6a83d584 100644 --- a/dojo/finding/deduplication.py +++ b/dojo/finding/deduplication.py @@ -1,4 +1,5 @@ import logging +from collections.abc import Iterator from operator import attrgetter import hyperlink @@ -127,7 +128,7 @@ def set_duplicate(new_finding, existing_finding): msg = "Can not add duplicate to itself" raise Exception(msg) if is_duplicate_reopen(new_finding, existing_finding): - msg = "Found a regression. Ignore this so that a new duplicate chain can be made" + msg = "Found a regression where a duplicate of a mitigated finding is found. We do not reopen this, but create a new duplicate chain as per @PR 9558: Deduplication: Do not reopen original finding" raise Exception(msg) if new_finding.duplicate and finding_mitigated(existing_finding): msg = "Skip this finding as we do not want to attach a new duplicate to a mitigated finding" @@ -174,17 +175,6 @@ def finding_not_human_set_status(finding: Finding) -> bool: return finding.out_of_scope is False and finding.false_p is False -def set_duplicate_reopen(new_finding, existing_finding): - logger.debug("duplicate reopen existing finding") - existing_finding.mitigated = new_finding.mitigated - existing_finding.is_mitigated = new_finding.is_mitigated - existing_finding.active = new_finding.active - existing_finding.verified = new_finding.verified - existing_finding.notes.create(author=existing_finding.reporter, - entry="This finding has been automatically re-opened as it was found in recent scans.") - existing_finding.save() - - def is_deduplication_on_engagement_mismatch(new_finding, to_duplicate_finding): if new_finding.test.engagement != to_duplicate_finding.test.engagement: deduplication_mismatch = new_finding.test.engagement.deduplication_on_engagement \ @@ -244,59 +234,123 @@ def are_endpoints_duplicates(new_finding, to_duplicate_finding): return False -def build_dedupe_scope_queryset(test): - scope_on_engagement = test.engagement.deduplication_on_engagement - if scope_on_engagement: - scope_q = Q(test__engagement=test.engagement) - else: - # Product scope limited to current product, but exclude engagements that opted into engagement-scoped dedupe - scope_q = Q(test__engagement__product=test.engagement.product) & ( - Q(test__engagement=test.engagement) - | Q(test__engagement__deduplication_on_engagement=False) - ) +def build_candidate_scope_queryset(test, mode="deduplication", service=None): + """ + Build a queryset for candidate finding. + + Args: + test: The test to scope from + mode: "deduplication" (can match across tests) or "reimport" (same test only) + service: Optional service filter (for deduplication mode, not used for reimport since service is in hash) + + """ + if mode == "reimport": + # For reimport, only filter by test. Service filtering is not needed because + # service is included in hash_code calculation (HASH_CODE_FIELDS_ALWAYS = ["service"]), + # so matching by hash_code automatically ensures correct service match. + queryset = Finding.objects.filter(test=test) + else: # deduplication mode + scope_on_engagement = test.engagement.deduplication_on_engagement + if scope_on_engagement: + scope_q = Q(test__engagement=test.engagement) + else: + # Product scope limited to current product, but exclude engagements that opted into engagement-scoped dedupe + scope_q = Q(test__engagement__product=test.engagement.product) & ( + Q(test__engagement=test.engagement) + | Q(test__engagement__deduplication_on_engagement=False) + ) + queryset = Finding.objects.filter(scope_q) + + # Base prefetches for both modes + prefetch_list = ["endpoints", "vulnerability_id_set", "found_by"] + + # Additional prefetches for reimport mode + if mode == "reimport": + prefetch_list.extend([ + "status_finding", + "status_finding__endpoint", + ]) return ( - Finding.objects.filter(scope_q) + queryset .select_related("test", "test__engagement", "test__test_type") - .prefetch_related("endpoints", "found_by") + .prefetch_related(*prefetch_list) ) -def find_candidates_for_deduplication_hash(test, findings): - base_queryset = build_dedupe_scope_queryset(test) +def find_candidates_for_deduplication_hash(test, findings, mode="deduplication", service=None): + """ + Find candidates by hash_code. Works for both deduplication and reimport. + + Args: + test: The test to scope from + findings: List of findings to find candidates for + mode: "deduplication" or "reimport" + service: Optional service filter (for deduplication mode, not used for reimport since service is in hash) + + """ + base_queryset = build_candidate_scope_queryset(test, mode=mode, service=service) hash_codes = {f.hash_code for f in findings if getattr(f, "hash_code", None) is not None} if not hash_codes: return {} - existing_qs = ( - base_queryset.filter(hash_code__in=hash_codes) - .exclude(hash_code=None) - .exclude(duplicate=True) - .order_by("id") - ) + + existing_qs = base_queryset.filter(hash_code__in=hash_codes).exclude(hash_code=None) + if mode == "deduplication": + existing_qs = existing_qs.exclude(duplicate=True) + existing_qs = existing_qs.order_by("id") + existing_by_hash = {} for ef in existing_qs: existing_by_hash.setdefault(ef.hash_code, []).append(ef) - deduplicationLogger.debug(f"Found {len(existing_by_hash)} existing findings by hash codes") + + log_msg = "for reimport" if mode == "reimport" else "" + deduplicationLogger.debug(f"Found {len(existing_by_hash)} existing findings by hash codes {log_msg}") return existing_by_hash -def find_candidates_for_deduplication_unique_id(test, findings): - base_queryset = build_dedupe_scope_queryset(test) +def find_candidates_for_deduplication_unique_id(test, findings, mode="deduplication", service=None): + """ + Find candidates by unique_id_from_tool. Works for both deduplication and reimport. + + Args: + test: The test to scope from + findings: List of findings to find candidates for + mode: "deduplication" or "reimport" + service: Optional service filter (for deduplication mode, not used for reimport since service is in hash) + + """ + base_queryset = build_candidate_scope_queryset(test, mode=mode, service=service) unique_ids = {f.unique_id_from_tool for f in findings if getattr(f, "unique_id_from_tool", None) is not None} if not unique_ids: return {} - existing_qs = base_queryset.filter(unique_id_from_tool__in=unique_ids).exclude(unique_id_from_tool=None).exclude(duplicate=True).order_by("id") + + existing_qs = base_queryset.filter(unique_id_from_tool__in=unique_ids).exclude(unique_id_from_tool=None) + if mode == "deduplication": + existing_qs = existing_qs.exclude(duplicate=True) # unique_id_from_tool can only apply to the same test_type because it is parser dependent - existing_qs = existing_qs.filter(test__test_type=test.test_type) + existing_qs = existing_qs.filter(test__test_type=test.test_type).order_by("id") + existing_by_uid = {} for ef in existing_qs: existing_by_uid.setdefault(ef.unique_id_from_tool, []).append(ef) - deduplicationLogger.debug(f"Found {len(existing_by_uid)} existing findings by unique IDs") + + log_msg = "for reimport" if mode == "reimport" else "" + deduplicationLogger.debug(f"Found {len(existing_by_uid)} existing findings by unique IDs {log_msg}") return existing_by_uid -def find_candidates_for_deduplication_uid_or_hash(test, findings): - base_queryset = build_dedupe_scope_queryset(test) +def find_candidates_for_deduplication_uid_or_hash(test, findings, mode="deduplication", service=None): + """ + Find candidates by unique_id_from_tool or hash_code. Works for both deduplication and reimport. + + Args: + test: The test to scope from + findings: List of findings to find candidates for + mode: "deduplication" or "reimport" + service: Optional service filter (for deduplication mode, not used for reimport since service is in hash) + + """ + base_queryset = build_candidate_scope_queryset(test, mode=mode, service=service) hash_codes = {f.hash_code for f in findings if getattr(f, "hash_code", None) is not None} unique_ids = {f.unique_id_from_tool for f in findings if getattr(f, "unique_id_from_tool", None) is not None} if not hash_codes and not unique_ids: @@ -310,7 +364,11 @@ def find_candidates_for_deduplication_uid_or_hash(test, findings): uid_q = Q(unique_id_from_tool__isnull=False, unique_id_from_tool__in=unique_ids) & Q(test__test_type=test.test_type) cond |= uid_q - existing_qs = base_queryset.filter(cond).exclude(duplicate=True).order_by("id") + existing_qs = base_queryset.filter(cond) + if mode == "deduplication": + # reimport matching will match against duplicates, import/deduplication doesn't. + existing_qs = existing_qs.exclude(duplicate=True) + existing_qs = existing_qs.order_by("id") existing_by_hash = {} existing_by_uid = {} @@ -319,13 +377,15 @@ def find_candidates_for_deduplication_uid_or_hash(test, findings): existing_by_hash.setdefault(ef.hash_code, []).append(ef) if ef.unique_id_from_tool is not None: existing_by_uid.setdefault(ef.unique_id_from_tool, []).append(ef) - deduplicationLogger.debug(f"Found {len(existing_by_uid)} existing findings by unique IDs") - deduplicationLogger.debug(f"Found {len(existing_by_hash)} existing findings by hash codes") + + log_msg = "for reimport" if mode == "reimport" else "" + deduplicationLogger.debug(f"Found {len(existing_by_uid)} existing findings by unique IDs {log_msg}") + deduplicationLogger.debug(f"Found {len(existing_by_hash)} existing findings by hash codes {log_msg}") return existing_by_uid, existing_by_hash def find_candidates_for_deduplication_legacy(test, findings): - base_queryset = build_dedupe_scope_queryset(test) + base_queryset = build_candidate_scope_queryset(test, mode="deduplication") titles = {f.title for f in findings if getattr(f, "title", None)} cwes = {f.cwe for f in findings if getattr(f, "cwe", 0)} cwes.discard(0) @@ -347,6 +407,52 @@ def find_candidates_for_deduplication_legacy(test, findings): return by_title, by_cwe +# TODO: should we align this with deduplication? +def find_candidates_for_reimport_legacy(test, findings, service=None): + """ + Find all existing findings in the test that match any of the given findings by title and severity. + Used for batch reimport to avoid 1+N query problem. + Legacy reimport matches by title (case-insensitive), severity, and numerical_severity. + Note: This function is kept separate because legacy reimport has fundamentally different matching logic + than legacy deduplication (title+severity vs title+CWE). + Note: service parameter is kept for backward compatibility but not used since service is in hash_code. + """ + base_queryset = build_candidate_scope_queryset(test, mode="reimport", service=None) + + # Collect all unique title/severity combinations + title_severity_pairs = set() + for finding in findings: + if finding.title: + title_severity_pairs.add(( + finding.title.lower(), # Case-insensitive matching + finding.severity, + Finding.get_numerical_severity(finding.severity), + )) + + if not title_severity_pairs: + return {} + + # Build query to find all matching findings + conditions = Q() + for title_lower, severity, numerical_severity in title_severity_pairs: + conditions |= ( + Q(title__iexact=title_lower) & + Q(severity=severity) & + Q(numerical_severity=numerical_severity) + ) + + existing_qs = base_queryset.filter(conditions).order_by("id") + + # Build dictionary keyed by (title_lower, severity) for quick lookup + existing_by_key = {} + for ef in existing_qs: + key = (ef.title.lower(), ef.severity) + existing_by_key.setdefault(key, []).append(ef) + + deduplicationLogger.debug(f"Found {sum(len(v) for v in existing_by_key.values())} existing findings by legacy matching for reimport") + return existing_by_key + + def _is_candidate_older(new_finding, candidate): # Ensure the newer finding is marked as duplicate of the older finding is_older = candidate.id < new_finding.id @@ -355,9 +461,9 @@ def _is_candidate_older(new_finding, candidate): return is_older -def match_hash_candidate(new_finding, candidates_by_hash): +def get_matches_from_hash_candidates(new_finding, candidates_by_hash) -> Iterator[Finding]: if new_finding.hash_code is None: - return None + return possible_matches = candidates_by_hash.get(new_finding.hash_code, []) deduplicationLogger.debug(f"Finding {new_finding.id}: Found {len(possible_matches)} findings with same hash_code, ids={[(c.id, c.hash_code) for c in possible_matches]}") @@ -368,13 +474,12 @@ def match_hash_candidate(new_finding, candidates_by_hash): deduplicationLogger.debug("deduplication_on_engagement_mismatch, skipping dedupe.") continue if are_endpoints_duplicates(new_finding, candidate): - return candidate - return None + yield candidate -def match_unique_id_candidate(new_finding, candidates_by_uid): +def get_matches_from_unique_id_candidates(new_finding, candidates_by_uid) -> Iterator[Finding]: if new_finding.unique_id_from_tool is None: - return None + return possible_matches = candidates_by_uid.get(new_finding.unique_id_from_tool, []) deduplicationLogger.debug(f"Finding {new_finding.id}: Found {len(possible_matches)} findings with same unique_id_from_tool, ids={[(c.id, c.unique_id_from_tool) for c in possible_matches]}") @@ -385,11 +490,10 @@ def match_unique_id_candidate(new_finding, candidates_by_uid): if is_deduplication_on_engagement_mismatch(new_finding, candidate): deduplicationLogger.debug("deduplication_on_engagement_mismatch, skipping dedupe.") continue - return candidate - return None + yield candidate -def match_uid_or_hash_candidate(new_finding, candidates_by_uid, candidates_by_hash): +def get_matches_from_uid_or_hash_candidates(new_finding, candidates_by_uid, candidates_by_hash) -> Iterator[Finding]: # Combine UID and hash candidates and walk oldest-first uid_list = candidates_by_uid.get(new_finding.unique_id_from_tool, []) if new_finding.unique_id_from_tool is not None else [] hash_list = candidates_by_hash.get(new_finding.hash_code, []) if new_finding.hash_code is not None else [] @@ -404,15 +508,15 @@ def match_uid_or_hash_candidate(new_finding, candidates_by_uid, candidates_by_ha continue if is_deduplication_on_engagement_mismatch(new_finding, candidate): deduplicationLogger.debug("deduplication_on_engagement_mismatch, skipping dedupe.") - return None + continue if are_endpoints_duplicates(new_finding, candidate): deduplicationLogger.debug("UID_OR_HASH: endpoints match, returning candidate %s with test_type %s unique_id_from_tool %s hash_code %s", candidate.id, candidate.test.test_type, candidate.unique_id_from_tool, candidate.hash_code) - return candidate - deduplicationLogger.debug("UID_OR_HASH: endpoints mismatch, skipping candidate %s", candidate.id) - return None + yield candidate + else: + deduplicationLogger.debug("UID_OR_HASH: endpoints mismatch, skipping candidate %s", candidate.id) -def match_legacy_candidate(new_finding, candidates_by_title, candidates_by_cwe): +def get_matches_from_legacy_candidates(new_finding, candidates_by_title, candidates_by_cwe) -> Iterator[Finding]: # --------------------------------------------------------- # 1) Collects all the findings that have the same: # (title and static_finding and dynamic_finding) @@ -469,8 +573,7 @@ def match_legacy_candidate(new_finding, candidates_by_title, candidates_by_cwe): + " flag_endpoints: " + str(flag_endpoints) + " flag_line_path:" + str(flag_line_path) + " flag_hash:" + str(flag_hash)) if (flag_endpoints or flag_line_path) and flag_hash: - return candidate - return None + yield candidate def _dedupe_batch_hash_code(findings): @@ -482,10 +585,10 @@ def _dedupe_batch_hash_code(findings): return for new_finding in findings: deduplicationLogger.debug(f"deduplication start for finding {new_finding.id} with DEDUPE_ALGO_HASH_CODE") - match = match_hash_candidate(new_finding, candidates_by_hash) - if match: + for match in get_matches_from_hash_candidates(new_finding, candidates_by_hash): try: set_duplicate(new_finding, match) + break except Exception as e: deduplicationLogger.debug(str(e)) @@ -499,12 +602,14 @@ def _dedupe_batch_unique_id(findings): return for new_finding in findings: deduplicationLogger.debug(f"deduplication start for finding {new_finding.id} with DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL") - match = match_unique_id_candidate(new_finding, candidates_by_uid) - if match: + for match in get_matches_from_unique_id_candidates(new_finding, candidates_by_uid): + deduplicationLogger.debug(f"Trying to deduplicate finding {new_finding.id} against candidate {match.id}") try: set_duplicate(new_finding, match) + deduplicationLogger.debug(f"Successfully deduplicated finding {new_finding.id} against candidate {match.id}") + break except Exception as e: - deduplicationLogger.debug(str(e)) + deduplicationLogger.debug(f"Exception when deduplicating finding {new_finding.id} against candidate {match.id}: {e!s}") def _dedupe_batch_uid_or_hash(findings): @@ -520,13 +625,12 @@ def _dedupe_batch_uid_or_hash(findings): if new_finding.duplicate: continue - match = match_uid_or_hash_candidate(new_finding, candidates_by_uid, existing_by_hash) - if match: + for match in get_matches_from_uid_or_hash_candidates(new_finding, candidates_by_uid, existing_by_hash): try: set_duplicate(new_finding, match) + break except Exception as e: deduplicationLogger.debug(str(e)) - continue def _dedupe_batch_legacy(findings): @@ -538,10 +642,10 @@ def _dedupe_batch_legacy(findings): return for new_finding in findings: deduplicationLogger.debug(f"deduplication start for finding {new_finding.id} with DEDUPE_ALGO_LEGACY") - match = match_legacy_candidate(new_finding, candidates_by_title, candidates_by_cwe) - if match: + for match in get_matches_from_legacy_candidates(new_finding, candidates_by_title, candidates_by_cwe): try: set_duplicate(new_finding, match) + break except Exception as e: deduplicationLogger.debug(str(e)) diff --git a/dojo/finding/helper.py b/dojo/finding/helper.py index a0cf29f2120..8b829455d21 100644 --- a/dojo/finding/helper.py +++ b/dojo/finding/helper.py @@ -17,7 +17,7 @@ import dojo.risk_acceptance.helper as ra_helper from dojo.celery import app from dojo.decorators import dojo_async_task, dojo_model_from_id, dojo_model_to_id -from dojo.endpoint.utils import save_endpoints_to_add +from dojo.endpoint.utils import endpoint_get_or_create, save_endpoints_to_add from dojo.file_uploads.helper import delete_related_files from dojo.finding.deduplication import ( dedupe_batch_of_findings, @@ -34,7 +34,6 @@ System_Settings, Test, Vulnerability_Id, - Vulnerability_Id_Template, ) from dojo.notes.helper import delete_related_notes from dojo.notifications.helper import create_notification @@ -767,7 +766,10 @@ def add_endpoints(new_finding, form): added_endpoints = save_endpoints_to_add(form.endpoints_to_add_list, new_finding.test.engagement.product) endpoint_ids = [endpoint.id for endpoint in added_endpoints] - new_finding.endpoints.set(form.cleaned_data["endpoints"] | Endpoint.objects.filter(id__in=endpoint_ids)) + # Merge form endpoints with existing endpoints (don't replace) + form_endpoints = form.cleaned_data.get("endpoints", Endpoint.objects.none()) + new_endpoints = Endpoint.objects.filter(id__in=endpoint_ids) + new_finding.endpoints.set(form_endpoints | new_endpoints | new_finding.endpoints.all()) for endpoint in new_finding.endpoints.all(): _eps, _created = Endpoint_Status.objects.get_or_create( @@ -775,13 +777,23 @@ def add_endpoints(new_finding, form): endpoint=endpoint, defaults={"date": form.cleaned_data["date"] or timezone.now()}) -def save_vulnerability_ids(finding, vulnerability_ids): +def sanitize_vulnerability_ids(vulnerability_ids) -> None: + """Remove undisired vulnerability id values""" + vulnerability_ids = [x for x in vulnerability_ids if x.strip()] + + +def save_vulnerability_ids(finding, vulnerability_ids, *, delete_existing: bool = True): # Remove duplicates vulnerability_ids = list(dict.fromkeys(vulnerability_ids)) - # Remove old vulnerability ids - Vulnerability_Id.objects.filter(finding=finding).delete() + # Remove old vulnerability ids if requested + # Callers can set delete_existing=False when they know there are no existing IDs + # to avoid an unnecessary delete query (e.g., for new findings) + if delete_existing: + Vulnerability_Id.objects.filter(finding=finding).delete() + # Remove undisired vulnerability ids + sanitize_vulnerability_ids(vulnerability_ids) # Save new vulnerability ids # Using bulk create throws Django 50 warnings about unsaved models... for vulnerability_id in vulnerability_ids: @@ -795,22 +807,169 @@ def save_vulnerability_ids(finding, vulnerability_ids): def save_vulnerability_ids_template(finding_template, vulnerability_ids): - # Remove duplicates - vulnerability_ids = list(dict.fromkeys(vulnerability_ids)) - - # Remove old vulnerability ids - Vulnerability_Id_Template.objects.filter(finding_template=finding_template).delete() + """Save vulnerability IDs as newline-separated string in TextField.""" + # Remove duplicates and empty strings + vulnerability_ids = list(dict.fromkeys([vid.strip() for vid in vulnerability_ids if vid.strip()])) - # Save new vulnerability ids - for vulnerability_id in vulnerability_ids: - Vulnerability_Id_Template(finding_template=finding_template, vulnerability_id=vulnerability_id).save() + # Save as newline-separated string + finding_template.vulnerability_ids_text = "\n".join(vulnerability_ids) if vulnerability_ids else None - # Set CVE + # Set CVE for backward compatibility if vulnerability_ids: finding_template.cve = vulnerability_ids[0] else: finding_template.cve = None + finding_template.save() + + +def save_endpoints_template(finding_template, endpoint_urls): + """Save endpoint URLs as newline-separated string in TextField.""" + # Remove duplicates and empty strings + endpoint_urls = list(dict.fromkeys([url.strip() for url in endpoint_urls if url.strip()])) + # Save as newline-separated string + finding_template.endpoints_text = "\n".join(endpoint_urls) if endpoint_urls else None + finding_template.save() + + +def copy_template_fields_to_finding( + finding, + template, + form_data=None, + user=None, + *, + copy_vulnerability_ids=True, + copy_endpoints=True, + copy_notes=True, +): + """ + Copy fields from Finding_Template to Finding. + + Args: + finding: Finding instance to update + template: Finding_Template instance (source) + form_data: Optional dict of form cleaned_data (if provided, uses form values instead of template) + user: User instance (required for notes) + copy_vulnerability_ids: Whether to copy vulnerability IDs (default True) + copy_endpoints: Whether to copy endpoints (default True) + copy_notes: Whether to copy notes (default True) + + """ + # Helper to get value from form_data or template + def get_value(field_name, default=None): + if form_data and field_name in form_data: + value = form_data.get(field_name) + # Handle None checks for boolean/optional fields + if value is not None or field_name not in form_data: + return value + return getattr(template, field_name, default) + + # Copy CVSS fields + cvssv3 = get_value("cvssv3") + if cvssv3: + finding.cvssv3 = cvssv3 + cvssv3_score = get_value("cvssv3_score") + if cvssv3_score is not None: + finding.cvssv3_score = cvssv3_score + cvssv4 = get_value("cvssv4") + if cvssv4: + finding.cvssv4 = cvssv4 + cvssv4_score = get_value("cvssv4_score") + if cvssv4_score is not None: + finding.cvssv4_score = cvssv4_score + + # Copy remediation planning fields + fix_available = get_value("fix_available") + if fix_available is not None: + finding.fix_available = fix_available + fix_version = get_value("fix_version") + if fix_version: + finding.fix_version = fix_version + planned_remediation_version = get_value("planned_remediation_version") + if planned_remediation_version: + finding.planned_remediation_version = planned_remediation_version + effort_for_fixing = get_value("effort_for_fixing") + if effort_for_fixing: + finding.effort_for_fixing = effort_for_fixing + + # Copy technical details fields + steps_to_reproduce = get_value("steps_to_reproduce") + if steps_to_reproduce: + finding.steps_to_reproduce = steps_to_reproduce + severity_justification = get_value("severity_justification") + if severity_justification: + finding.severity_justification = severity_justification + component_name = get_value("component_name") + if component_name: + finding.component_name = component_name + component_version = get_value("component_version") + if component_version: + finding.component_version = component_version + + # Copy vulnerability IDs + if copy_vulnerability_ids: + if form_data and "vulnerability_ids" in form_data: + # Split form data (space or newline separated) + vulnerability_ids = form_data["vulnerability_ids"] + if isinstance(vulnerability_ids, str): + vulnerability_ids = vulnerability_ids.split() + save_vulnerability_ids(finding, vulnerability_ids, delete_existing=True) + elif template.vulnerability_ids: + save_vulnerability_ids(finding, template.vulnerability_ids, delete_existing=False) + + # Copy endpoints + if copy_endpoints: + endpoint_urls = None + if form_data and form_data.get("endpoints"): + # Parse from form data (newline-separated string) + endpoint_urls = [url.strip() for url in form_data["endpoints"].split("\n") if url.strip()] + elif template.endpoints: + # Parse from template (list or newline-separated string) + if isinstance(template.endpoints, list): + endpoint_urls = template.endpoints + else: + endpoint_urls = [url.strip() for url in template.endpoints.split("\n") if url.strip()] + + if endpoint_urls: + product = finding.test.engagement.product + for endpoint_url in endpoint_urls: + try: + endpoint = Endpoint.from_uri(endpoint_url) + ep, _ = endpoint_get_or_create( + protocol=endpoint.protocol, + host=endpoint.host, + port=endpoint.port, + path=endpoint.path, + query=endpoint.query, + fragment=endpoint.fragment, + product=product, + ) + Endpoint_Status.objects.get_or_create( + finding=finding, + endpoint=ep, + defaults={"date": finding.date or timezone.now()}, + ) + except Exception as e: + logger.warning(f"Failed to parse endpoint URL '{endpoint_url}': {e}") + + # Copy notes + if copy_notes and user: + notes_content = None + if form_data and form_data.get("notes"): + notes_content = form_data["notes"] + elif template.notes: + notes_content = template.notes + + if notes_content: + note = Notes( + entry=notes_content, + author=user, + date=timezone.now(), + private=False, + ) + note.save() + finding.notes.add(note) + def normalize_datetime(value): """Ensure value is timezone-aware datetime.""" diff --git a/dojo/finding/urls.py b/dojo/finding/urls.py index 3b59624029a..fa442df384c 100644 --- a/dojo/finding/urls.py +++ b/dojo/finding/urls.py @@ -134,8 +134,6 @@ views.clear_finding_review, name="clear_finding_review"), re_path(r"^finding/(?P\d+)/copy$", views.copy_finding, name="copy_finding"), - re_path(r"^finding/(?P\d+)/apply_cwe$", - views.apply_template_cwe, name="apply_template_cwe"), re_path(r"^finding/(?P\d+)/mktemplate$", views.mktemplate, name="mktemplate"), re_path(r"^finding/(?P\d+)/find_template_to_apply$", views.find_template_to_apply, diff --git a/dojo/finding/views.py b/dojo/finding/views.py index 4e7100014af..999e15e6e47 100644 --- a/dojo/finding/views.py +++ b/dojo/finding/views.py @@ -9,17 +9,17 @@ from itertools import chain from pathlib import Path +import pghistory from django.conf import settings from django.contrib import messages from django.core import serializers from django.core.exceptions import PermissionDenied, ValidationError from django.db import models -from django.db.models import F, QuerySet, Value +from django.db.models import Case, F, QuerySet, Value, When from django.db.models.functions import Coalesce, ExtractDay, Length, TruncDate from django.db.models.query import Prefetch from django.http import Http404, HttpRequest, HttpResponse, HttpResponseRedirect, JsonResponse, StreamingHttpResponse from django.shortcuts import get_object_or_404, render -from django.template.defaultfilters import pluralize from django.urls import reverse from django.utils import formats, timezone from django.utils.safestring import mark_safe @@ -62,7 +62,6 @@ EditPlannedRemediationDateFindingForm, FindingBulkUpdateForm, FindingForm, - FindingFormID, FindingTemplateForm, GITHUBFindingForm, JIRAFindingForm, @@ -97,7 +96,6 @@ Test_Import, Test_Import_Finding_Action, User, - Vulnerability_Id_Template, ) from dojo.notifications.helper import create_notification from dojo.tag_utils import bulk_add_tags_to_instances @@ -111,13 +109,13 @@ add_external_issue, add_field_errors_to_response, add_success_message_to_response, - apply_cwe_to_template, calculate_grade, do_false_positive_history, get_page_items, get_page_items_and_count, get_return_url, get_system_setting, + get_visible_scan_types, get_words_for_field, match_finding_to_existing_findings, process_tag_notifications, @@ -315,6 +313,7 @@ def get_initial_context(self, request: HttpRequest): "enable_table_filtering": get_system_setting("enable_ui_table_based_searching"), "title_words": get_words_for_field(Finding, "title"), "component_words": get_words_for_field(Finding, "component_name"), + "visible_test_types": get_visible_scan_types(), } # Look to see if the product was used if product_id := self.get_product_id(): @@ -487,15 +486,6 @@ def get_credential_objects(self, finding: Finding): "cred_engagement": cred_engagement, } - def get_cwe_template(self, finding: Finding): - cwe_template = None - with contextlib.suppress(Finding_Template.DoesNotExist): - cwe_template = Finding_Template.objects.filter(cwe=finding.cwe).first() - - return { - "cwe_template": cwe_template, - } - def get_request_response(self, finding: Finding): request_response = None burp_request = None @@ -706,7 +696,6 @@ def get(self, request: HttpRequest, finding_id: int): # Add in the other extras context |= self.get_previous_and_next_findings(finding) context |= self.get_credential_objects(finding) - context |= self.get_cwe_template(finding) # Add in more of the other extras context |= self.get_request_response(finding) context |= self.get_similar_findings(request, finding) @@ -1349,31 +1338,6 @@ def reopen_finding(request, fid): return HttpResponseRedirect(reverse("view_finding", args=(finding.id,))) -@user_is_authorized(Finding, Permissions.Finding_Edit, "fid") -def apply_template_cwe(request, fid): - finding = get_object_or_404(Finding, id=fid) - if request.method == "POST": - form = FindingFormID(request.POST, instance=finding) - if form.is_valid(): - finding = apply_cwe_to_template(finding) - finding.save() - messages.add_message( - request, - messages.SUCCESS, - "Finding CWE template applied successfully.", - extra_tags="alert-success", - ) - return HttpResponseRedirect(reverse("view_finding", args=(fid,))) - messages.add_message( - request, - messages.ERROR, - "Unable to apply CWE template finding, please try again.", - extra_tags="alert-danger", - ) - return None - raise PermissionDenied - - @user_is_authorized(Finding, Permissions.Finding_Edit, "fid") def copy_finding(request, fid): finding = get_object_or_404(Finding, id=fid) @@ -1700,21 +1664,38 @@ def mktemplate(request, fid): title=finding.title, cwe=finding.cwe, cvssv3=finding.cvssv3, + cvssv3_score=finding.cvssv3_score, + cvssv4=finding.cvssv4, + cvssv4_score=finding.cvssv4_score, severity=finding.severity, description=finding.description, mitigation=finding.mitigation, impact=finding.impact, references=finding.references, numerical_severity=finding.numerical_severity, + fix_available=finding.fix_available, + fix_version=finding.fix_version, + planned_remediation_version=finding.planned_remediation_version, + effort_for_fixing=finding.effort_for_fixing, + steps_to_reproduce=finding.steps_to_reproduce, + severity_justification=finding.severity_justification, + component_name=finding.component_name, + component_version=finding.component_version, tags=finding.tags.all(), ) template.save() template.tags = finding.tags.all() + # Ensure template tags exist in Finding's tag model + # (They should already exist since they come from a finding, but ensure for consistency) + ensure_template_tags_in_finding_model(template) - for vulnerability_id in finding.vulnerability_ids: - Vulnerability_Id_Template( - finding_template=template, vulnerability_id=vulnerability_id, - ).save() + # Save vulnerability IDs using helper (handles both old and new format) + finding_helper.save_vulnerability_ids_template(template, finding.vulnerability_ids) + + # Copy endpoints if they exist + if finding.endpoints.exists(): + endpoint_urls = [str(ep) for ep in finding.endpoints.all()] + finding_helper.save_endpoints_template(template, endpoint_urls) messages.add_message( request, @@ -1756,9 +1737,20 @@ def find_template_to_apply(request, fid): cve_len=Length("cve"), order=models.Value(2, models.IntegerField()), ) ) - templates = templates_by_last_used.union(templates_by_cve).order_by( + union_queryset = templates_by_last_used.union(templates_by_cve).order_by( "order", "-last_used", ) + # Convert union queryset to regular queryset to avoid issues with distinct() in filters + # Get IDs from union queryset and create a new queryset filtered by those IDs + template_ids = list(union_queryset.values_list("id", flat=True)) + templates = Finding_Template.objects.filter(id__in=template_ids).annotate( + cve_len=Length("cve"), + order=Case( + *[When(id=template_id, then=models.Value(i + 1)) for i, template_id in enumerate(template_ids)], + default=models.Value(len(template_ids) + 1), + output_field=models.IntegerField(), + ), + ).order_by("order", "-last_used") templates = TemplateFindingFilter(request.GET, queryset=templates) paged_templates = get_page_items(request, templates.qs, 25) @@ -1788,12 +1780,72 @@ def find_template_to_apply(request, fid): def choose_finding_template_options(request, tid, fid): finding = get_object_or_404(Finding, id=fid) template = get_object_or_404(Finding_Template, id=tid) - data = finding.__dict__ - # Not sure what's going on here, just leave same as with django-tagging - data["tags"] = [tag.name for tag in template.tags.all()] + data = finding.__dict__.copy() + # Remove tags and other non-serializable fields + data.pop("tags", None) + data.pop("_state", None) + data.pop("_tags_tagulous", None) + + # Populate from template for fields that exist on template + template_fields = ["cvssv3", "cvssv3_score", "cvssv4", "cvssv4_score", + "fix_available", "fix_version", "planned_remediation_version", + "effort_for_fixing", "steps_to_reproduce", "severity_justification", + "component_name", "component_version", "notes"] + for field in template_fields: + if hasattr(template, field): + value = getattr(template, field) + if value is not None: + data[field] = value + + # Handle vulnerability_ids and endpoints (convert lists to strings) data["vulnerability_ids"] = "\n".join(finding.vulnerability_ids) + if hasattr(template, "endpoints") and template.endpoints: + endpoints_value = template.endpoints + if isinstance(endpoints_value, list): + data["endpoints"] = "\n".join(endpoints_value) + else: + data["endpoints"] = endpoints_value + + template_tag_names = [tag.name for tag in template.tags.all()] + # Add tags as comma-separated string for TagField + if template_tag_names: + data["tags"] = ", ".join(template_tag_names) form = ApplyFindingTemplateForm(data=data, template=template) + # Combine tags from both Finding_Template and Finding tag models + # This ensures we don't lose tags that exist on templates but may have been removed from findings + if "tags" in form.fields: + finding_tag_model = Finding.tags.tag_model + template_tag_model = Finding_Template.tags.tag_model + + # Get all tags from Finding_Template model + template_tags = set(template_tag_model.objects.values_list("name", flat=True)) + # Get all tags from Finding model + finding_tags = set(finding_tag_model.objects.values_list("name", flat=True)) + # Combine both sets to get all unique tag names + all_tag_names = template_tags | finding_tags + + # Ensure all tags from both models exist in Finding's tag model (where they'll be applied) + # Strictly speaking, creating tags here isn't necessary since TagField can create them on save, + # but it's the safest option to avoid tags getting lost or not getting rendered properly. + # This prevents tagulous from removing tags that only exist on templates and ensures + # TagField can display them correctly during form rendering. + # Store tag objects in a dict for reuse + tag_objects = {} + for tag_name in all_tag_names: + tag, _ = finding_tag_model.objects.get_or_create( + name=tag_name, + defaults={"name": tag_name, "protected": False}, + ) + tag_objects[tag_name] = tag + + # Update autocomplete_tags to include tags from both models + form.fields["tags"].autocomplete_tags = finding_tag_model.objects.all().order_by("name") + + # Set initial value using template tags (already created above) + if template_tag_names: + template_finding_tags = [tag_objects[tag_name] for tag_name in template_tag_names] + form.fields["tags"].initial = template_finding_tags product_tab = Product_Tab( finding.test.engagement.product, title="Finding Template Options", @@ -1823,6 +1875,8 @@ def apply_template_to_finding(request, fid, tid): if form.is_valid(): template.last_used = timezone.now() template.save() + + # Apply basic fields (existing) finding.title = form.cleaned_data["title"] finding.cwe = form.cleaned_data["cwe"] finding.severity = form.cleaned_data["severity"] @@ -1830,15 +1884,24 @@ def apply_template_to_finding(request, fid, tid): finding.mitigation = form.cleaned_data["mitigation"] finding.impact = form.cleaned_data["impact"] finding.references = form.cleaned_data["references"] - finding.last_reviewed = timezone.now() - finding.last_reviewed_by = request.user finding.tags = form.cleaned_data["tags"] - finding.cve = None - finding_helper.save_vulnerability_ids( - finding, form.cleaned_data["vulnerability_ids"].split(), + # Copy template fields (using centralized helper) + finding_helper.copy_template_fields_to_finding( + finding=finding, + template=template, + form_data=form.cleaned_data, + user=request.user, + copy_vulnerability_ids=True, + copy_endpoints=True, + copy_notes=True, ) + # Update review fields + finding.last_reviewed = timezone.now() + finding.last_reviewed_by = request.user + + # Save finding (this will trigger CVSS score computation if vectors are set) finding.save() else: messages.add_message( @@ -2115,6 +2178,28 @@ def export_templates_to_json(request): return HttpResponse(leads_as_json, content_type="json") +def ensure_template_tags_in_finding_model(template): + """ + Ensure all tags on a Finding_Template also exist in Finding's tag model. + This prevents tags from being lost when tagulous cleans up unused tags and ensures + tags can be properly applied when templates are used. + """ + if not template or not template.pk: + return + + finding_tag_model = Finding.tags.tag_model + + # Get all tag names from the template + template_tag_names = [tag.name for tag in template.tags.all()] + + # Ensure each tag exists in Finding's tag model + for tag_name in template_tag_names: + finding_tag_model.objects.get_or_create( + name=tag_name, + defaults={"name": tag_name, "protected": False}, + ) + + def apply_cwe_mitigation(apply_to_findings, template, *, update=True): count = 0 if apply_to_findings and template.template_match and template.cwe is not None: @@ -2187,28 +2272,27 @@ def add_template(request): if request.method == "POST": form = FindingTemplateForm(request.POST) if form.is_valid(): - apply_message = "" template = form.save(commit=False) template.numerical_severity = Finding.get_numerical_severity( template.severity, ) - template.save() + # Save vulnerability IDs using helper finding_helper.save_vulnerability_ids_template( template, form.cleaned_data["vulnerability_ids"].split(), ) + # Save endpoints using helper + if form.cleaned_data.get("endpoints"): + endpoint_urls = [url.strip() for url in form.cleaned_data["endpoints"].split("\n") if url.strip()] + finding_helper.save_endpoints_template(template, endpoint_urls) + template.save() form.save_m2m() - count = apply_cwe_mitigation( - form.cleaned_data["apply_to_findings"], template, - ) - if count > 0: - apply_message = ( - " and " + str(count) + pluralize(count, "finding,findings") + " " - ) + # Ensure template tags exist in Finding's tag model + ensure_template_tags_in_finding_model(template) messages.add_message( request, messages.SUCCESS, - "Template created successfully. " + apply_message, + "Template created successfully.", extra_tags="alert-success", ) return HttpResponseRedirect(reverse("templates")) @@ -2227,9 +2311,17 @@ def add_template(request): @user_has_global_permission(Permissions.Finding_Edit) def edit_template(request, tid): template = get_object_or_404(Finding_Template, id=tid) + initial_data = {"vulnerability_ids": "\n".join(template.vulnerability_ids)} + # Add endpoints to initial data if they exist + if hasattr(template, "endpoints") and template.endpoints: + endpoints_value = template.endpoints + if isinstance(endpoints_value, list): + initial_data["endpoints"] = "\n".join(endpoints_value) + else: + initial_data["endpoints"] = endpoints_value form = FindingTemplateForm( instance=template, - initial={"vulnerability_ids": "\n".join(template.vulnerability_ids)}, + initial=initial_data, ) if request.method == "POST": @@ -2239,21 +2331,23 @@ def edit_template(request, tid): template.numerical_severity = Finding.get_numerical_severity( template.severity, ) + # Save vulnerability IDs using helper finding_helper.save_vulnerability_ids_template( template, form.cleaned_data["vulnerability_ids"].split(), ) + # Save endpoints using helper + if form.cleaned_data.get("endpoints"): + endpoint_urls = [url.strip() for url in form.cleaned_data["endpoints"].split("\n") if url.strip()] + finding_helper.save_endpoints_template(template, endpoint_urls) template.save() form.save_m2m() - - count = apply_cwe_mitigation( - form.cleaned_data["apply_to_findings"], template, - ) - apply_message = " and " + str(count) + " " + pluralize(count, "finding,findings") + " " if count > 0 else "" + # Ensure template tags exist in Finding's tag model + ensure_template_tags_in_finding_model(template) messages.add_message( request, messages.SUCCESS, - "Template " + apply_message + "updated successfully.", + "Template updated successfully.", extra_tags="alert-success", ) return HttpResponseRedirect(reverse("templates")) @@ -2264,14 +2358,12 @@ def edit_template(request, tid): extra_tags="alert-danger", ) - count = apply_cwe_mitigation(apply_to_findings=True, template=template, update=False) add_breadcrumb(title="Edit Template", top_level=False, request=request) return render( request, "dojo/add_template.html", { "form": form, - "count": count, "name": "Edit Template", "template": template, }, @@ -2538,6 +2630,393 @@ def merge_finding_product(request, pid): # bulk update and delete are combined, so we can't have the nice user_is_authorized decorator + + +def _bulk_delete_findings(request, pid, form, finding_to_update, finds, total_find_count): + """Helper function to handle bulk deletion of findings.""" + if form.is_valid() and finding_to_update: + if pid is not None: + product = get_object_or_404(Product, id=pid) + user_has_permission_or_403( + request.user, product, Permissions.Finding_Delete, + ) + + finds = get_authorized_findings( + Permissions.Finding_Delete, finds, + ).distinct() + + skipped_find_count = total_find_count - finds.count() + deleted_find_count = finds.count() + + for find in finds: + find.delete() + + if skipped_find_count > 0: + add_error_message_to_response( + f"Skipped deletion of {skipped_find_count} findings because you are not authorized.", + ) + + if deleted_find_count > 0: + messages.add_message( + request, + messages.SUCCESS, + f"Bulk delete of {deleted_find_count} findings was successful.", + extra_tags="alert-success", + ) + + +def _bulk_update_finding_status_and_severity(finds, form, request, system_settings, prods, now): + """Helper function to handle status and severity updates for findings.""" + skipped_duplicate_count = 0 + actually_updated_count = 0 + + if form.cleaned_data["severity"] or form.cleaned_data["status"]: + for find in finds: + old_find = copy.deepcopy(find) + + if form.cleaned_data["severity"]: + find.severity = form.cleaned_data["severity"] + find.numerical_severity = Finding.get_numerical_severity( + form.cleaned_data["severity"], + ) + find.last_reviewed = now + find.last_reviewed_by = request.user + + if form.cleaned_data["status"]: + # logger.debug('setting status from bulk edit form: %s', form) + # Check if finding is duplicate and user wants to set active/verified + if find.duplicate and (form.cleaned_data["active"] or form.cleaned_data["verified"]): + # Skip active/verified but allow other status changes + skipped_duplicate_count += 1 + # Set other fields but not active/verified + find.false_p = form.cleaned_data["false_p"] + find.out_of_scope = form.cleaned_data["out_of_scope"] + find.is_mitigated = form.cleaned_data["is_mitigated"] + find.under_review = form.cleaned_data["under_review"] + else: + # Apply all status changes normally + find.active = form.cleaned_data["active"] + find.verified = form.cleaned_data["verified"] + find.false_p = form.cleaned_data["false_p"] + find.out_of_scope = form.cleaned_data["out_of_scope"] + find.is_mitigated = form.cleaned_data["is_mitigated"] + find.under_review = form.cleaned_data["under_review"] + find.last_reviewed = timezone.now() + find.last_reviewed_by = request.user + + # use super to avoid all custom logic in our overriden save method + # it will trigger the pre_save signal + find.save_no_options() + actually_updated_count += 1 + + if system_settings.false_positive_history: + # If finding is being marked as false positive + if find.false_p: + do_false_positive_history(find) + + # If finding was a false positive and is being reactivated: retroactively reactivates all equal findings + elif old_find.false_p and not find.false_p: + if system_settings.retroactive_false_positive_history: + logger.debug("FALSE_POSITIVE_HISTORY: Reactivating existing findings based on: %s", find) + + existing_fp_findings = match_finding_to_existing_findings( + find, product=find.test.engagement.product, + ).filter(false_p=True) + + for fp in existing_fp_findings: + logger.debug("FALSE_POSITIVE_HISTORY: Reactivating false positive %i: %s", fp.id, fp) + fp.active = find.active + fp.verified = find.verified + fp.false_p = False + fp.out_of_scope = find.out_of_scope + fp.is_mitigated = find.is_mitigated + fp.save_no_options() + + for prod in prods: + calculate_grade(prod) + + if skipped_duplicate_count > 0: + messages.add_message( + request, + messages.WARNING, + f"Skipped status update of {skipped_duplicate_count} duplicate findings. Duplicate findings cannot be active or verified.", + extra_tags="alert-warning", + ) + + return skipped_duplicate_count, actually_updated_count + + +def _bulk_update_simple_fields(finds, form): + """Helper function to handle simple field updates (date, planned_remediation_date, etc.).""" + if form.cleaned_data["date"]: + for finding in finds: + finding.date = form.cleaned_data["date"] + finding.save_no_options() + + if form.cleaned_data["planned_remediation_date"]: + for finding in finds: + finding.planned_remediation_date = form.cleaned_data[ + "planned_remediation_date" + ] + finding.save_no_options() + + if form.cleaned_data["planned_remediation_version"]: + for finding in finds: + finding.planned_remediation_version = form.cleaned_data[ + "planned_remediation_version" + ] + finding.save_no_options() + + +def _bulk_update_risk_acceptance(finds, form, request, prods): + """Helper function to handle risk acceptance updates.""" + skipped_risk_accept_count = 0 + skipped_active_risk_accept_count = 0 + + if form.cleaned_data["risk_acceptance"]: + for finding in finds: + if finding.active: + skipped_active_risk_accept_count += 1 + # Allow risk acceptance for inactive findings (whether duplicate or not) + elif form.cleaned_data["risk_accept"]: + if ( + not finding.test.engagement.product.enable_simple_risk_acceptance + ): + skipped_risk_accept_count += 1 + else: + ra_helper.simple_risk_accept(request.user, finding) + elif form.cleaned_data["risk_unaccept"]: + ra_helper.risk_unaccept(request.user, finding) + + for prod in prods: + calculate_grade(prod) + + if skipped_risk_accept_count > 0: + messages.add_message( + request, + messages.WARNING, + (f"Skipped simple risk acceptance of {skipped_risk_accept_count} findings, " + "simple risk acceptance is disabled on the related products"), + extra_tags="alert-warning", + ) + + if skipped_active_risk_accept_count > 0: + messages.add_message( + request, + messages.WARNING, + f"Skipped risk acceptance of {skipped_active_risk_accept_count} active findings. Active findings cannot be risk accepted.", + extra_tags="alert-warning", + ) + + return skipped_risk_accept_count, skipped_active_risk_accept_count + + +def _bulk_update_finding_groups(finds, form): + """Helper function to handle finding group operations.""" + return_url = None + + if form.cleaned_data["finding_group_create"]: + logger.debug("finding_group_create checked!") + finding_group_name = form.cleaned_data["finding_group_create_name"] + logger.debug("finding_group_create_name: %s", finding_group_name) + finding_group, added, skipped = finding_helper.create_finding_group( + finds, finding_group_name, + ) + + if added: + add_success_message_to_response( + f"Created finding group with {added} findings", + ) + return_url = reverse( + "view_finding_group", args=(finding_group.id,), + ) + + if skipped: + add_success_message_to_response( + f"Skipped {skipped} findings in group creation, findings already part of another group", + ) + + # refresh findings from db + finds = finds.all() + + if form.cleaned_data["finding_group_add"]: + logger.debug("finding_group_add checked!") + fgid = form.cleaned_data["add_to_finding_group_id"] + finding_group = Finding_Group.objects.get(id=fgid) + finding_group, added, skipped = finding_helper.add_to_finding_group( + finding_group, finds, + ) + + if added: + add_success_message_to_response( + f"Added {added} findings to finding group {finding_group.name}", + ) + return_url = reverse( + "view_finding_group", args=(finding_group.id,), + ) + + if skipped: + add_success_message_to_response( + f"Skipped {skipped} findings when adding to finding group {finding_group.name}, " + "findings already part of another group", + ) + + # refresh findings from db + finds = finds.all() + + if form.cleaned_data["finding_group_remove"]: + logger.debug("finding_group_remove checked!") + ( + finding_groups, + removed, + skipped, + ) = finding_helper.remove_from_finding_group(finds) + + if removed: + add_success_message_to_response( + "Removed {} findings from finding groups {}".format( + removed, + ",".join( + [ + finding_group.name + for finding_group in finding_groups + ], + ), + ), + ) + + if skipped: + add_success_message_to_response( + f"Skipped {skipped} findings when removing from any finding group, findings not part of any group", + ) + + # refresh findings from db + finds = finds.all() + + if form.cleaned_data["finding_group_by"]: + logger.debug("finding_group_by checked!") + logger.debug(form.cleaned_data) + finding_group_by_option = form.cleaned_data[ + "finding_group_by_option" + ] + logger.debug("finding_group_by_option: %s", finding_group_by_option) + + ( + finding_groups, + grouped, + skipped, + groups_created, + ) = finding_helper.group_findings_by(finds, finding_group_by_option) + + if grouped: + add_success_message_to_response( + f"Grouped {grouped} findings into {len(finding_groups)} ({groups_created} newly created) finding groups", + ) + + if skipped: + add_success_message_to_response( + f"Skipped {skipped} findings when grouping by {finding_group_by_option} as these findings " + "were already in an existing group", + ) + + # refresh findings from db + finds = finds.all() + + return return_url, finds + + +def _bulk_push_to_jira(finds, form, note): + """Helper function to handle JIRA push operations.""" + error_counts = defaultdict(lambda: 0) + success_count = 0 + finding_groups = set( # noqa: C401 + finding.finding_group + for finding in finds + if finding.has_finding_group + and ( + jira_helper.is_push_all_issues(finding) + or jira_helper.is_keep_in_sync_with_jira(finding) + or form.cleaned_data.get("push_to_jira") + ) + ) + logger.debug("finding_groups: %s", finding_groups) + for group in finding_groups: + if form.cleaned_data.get("push_to_jira"): + ( + can_be_pushed_to_jira, + error_message, + _error_code, + ) = jira_helper.can_be_pushed_to_jira(group) + if not can_be_pushed_to_jira: + error_counts[error_message] += 1 + jira_helper.log_jira_cannot_be_pushed_reason(error_message, group) + else: + logger.debug( + "pushing to jira from finding.finding_bulk_update_all()", + ) + jira_helper.push_to_jira(group) + success_count += 1 + + for error_message, error_count in error_counts.items(): + add_error_message_to_response(f"{error_count} finding groups could not be pushed to JIRA: {error_message}") + + if success_count > 0: + add_success_message_to_response(f"{success_count} finding groups pushed to JIRA successfully") + + # refresh from db + finds = finds.all() + + error_counts = defaultdict(lambda: 0) + success_count = 0 + for finding in finds: + tool_issue_updater.async_tool_issue_update(finding) + + # not sure yet if we want to support bulk unlink, so leave as commented out for now + # if form.cleaned_data['unlink_from_jira']: + # if finding.has_jira_issue: + # jira_helper.finding_unlink_jira(request, finding) + + # Because we never call finding.save() in a bulk update, we need to actually + # push the JIRA stuff here, rather than in finding.save() + # can't use helper as when push_all_jira_issues is True, + # the checkbox gets disabled and is always false + # push_to_jira = jira_helper.is_push_to_jira(new_finding, + # form.cleaned_data.get('push_to_jira')) + if ( + form.cleaned_data.get("push_to_jira") + or jira_helper.is_push_all_issues(finding) + or jira_helper.is_keep_in_sync_with_jira(finding) + ) and not finding.has_finding_group: + ( + can_be_pushed_to_jira, + error_message, + _error_code, + ) = jira_helper.can_be_pushed_to_jira(finding) + if finding.has_jira_group_issue and not finding.has_jira_issue: + error_message = ( + "finding already pushed as part of Finding Group" + ) + error_counts[error_message] += 1 + jira_helper.log_jira_cannot_be_pushed_reason(error_message, finding) + elif not can_be_pushed_to_jira: + error_counts[error_message] += 1 + jira_helper.log_jira_cannot_be_pushed_reason(error_message, finding) + else: + logger.debug( + "pushing to jira from finding.finding_bulk_update_all()", + ) + jira_helper.push_to_jira(finding) + if note is not None and isinstance(note, Notes): + jira_helper.add_comment(finding, note) + success_count += 1 + + for error_message, error_count in error_counts.items(): + add_error_message_to_response(f"{error_count} findings could not be pushed to JIRA: {error_message}") + + if success_count > 0: + add_success_message_to_response(f"{success_count} findings pushed to JIRA successfully") + + def finding_bulk_update_all(request, pid=None): system_settings = System_Settings.objects.get() @@ -2550,39 +3029,16 @@ def finding_bulk_update_all(request, pid=None): logger.debug("bulk 20") finding_to_update = request.POST.getlist("finding_to_update") + # Add pghistory context for audit trail (adds to existing middleware context) + pghistory.context( + source="bulk_edit", + finding_count=len(finding_to_update), + ) finds = Finding.objects.filter(id__in=finding_to_update).order_by("id") total_find_count = finds.count() prods = set(find.test.engagement.product for find in finds) # noqa: C401 if request.POST.get("delete_bulk_findings"): - if form.is_valid() and finding_to_update: - if pid is not None: - product = get_object_or_404(Product, id=pid) - user_has_permission_or_403( - request.user, product, Permissions.Finding_Delete, - ) - - finds = get_authorized_findings( - Permissions.Finding_Delete, finds, - ).distinct() - - skipped_find_count = total_find_count - finds.count() - deleted_find_count = finds.count() - - for find in finds: - find.delete() - - if skipped_find_count > 0: - add_error_message_to_response( - f"Skipped deletion of {skipped_find_count} findings because you are not authorized.", - ) - - if deleted_find_count > 0: - messages.add_message( - request, - messages.SUCCESS, - f"Bulk delete of {deleted_find_count} findings was successful.", - extra_tags="alert-success", - ) + _bulk_delete_findings(request, pid, form, finding_to_update, finds, total_find_count) elif form.is_valid() and finding_to_update: if pid is not None: product = get_object_or_404(Product, id=pid) @@ -2605,210 +3061,21 @@ def finding_bulk_update_all(request, pid=None): finds = prefetch_for_findings(finds) note = None - if form.cleaned_data["severity"] or form.cleaned_data["status"]: - for find in finds: - old_find = copy.deepcopy(find) - - if form.cleaned_data["severity"]: - find.severity = form.cleaned_data["severity"] - find.numerical_severity = Finding.get_numerical_severity( - form.cleaned_data["severity"], - ) - find.last_reviewed = now - find.last_reviewed_by = request.user - - if form.cleaned_data["status"]: - # logger.debug('setting status from bulk edit form: %s', form) - find.active = form.cleaned_data["active"] - find.verified = form.cleaned_data["verified"] - find.false_p = form.cleaned_data["false_p"] - find.out_of_scope = form.cleaned_data["out_of_scope"] - find.is_mitigated = form.cleaned_data["is_mitigated"] - find.under_review = form.cleaned_data["under_review"] - find.last_reviewed = timezone.now() - find.last_reviewed_by = request.user - - # use super to avoid all custom logic in our overriden save method - # it will trigger the pre_save signal - find.save_no_options() - - if system_settings.false_positive_history: - # If finding is being marked as false positive - if find.false_p: - do_false_positive_history(find) - - # If finding was a false positive and is being reactivated: retroactively reactivates all equal findings - elif old_find.false_p and not find.false_p: - if system_settings.retroactive_false_positive_history: - logger.debug("FALSE_POSITIVE_HISTORY: Reactivating existing findings based on: %s", find) - - existing_fp_findings = match_finding_to_existing_findings( - find, product=find.test.engagement.product, - ).filter(false_p=True) - - for fp in existing_fp_findings: - logger.debug("FALSE_POSITIVE_HISTORY: Reactivating false positive %i: %s", fp.id, fp) - fp.active = find.active - fp.verified = find.verified - fp.false_p = False - fp.out_of_scope = find.out_of_scope - fp.is_mitigated = find.is_mitigated - fp.save_no_options() - - for prod in prods: - calculate_grade(prod) - - if form.cleaned_data["date"]: - for finding in finds: - finding.date = form.cleaned_data["date"] - finding.save_no_options() - - if form.cleaned_data["planned_remediation_date"]: - for finding in finds: - finding.planned_remediation_date = form.cleaned_data[ - "planned_remediation_date" - ] - finding.save_no_options() - - if form.cleaned_data["planned_remediation_version"]: - for finding in finds: - finding.planned_remediation_version = form.cleaned_data[ - "planned_remediation_version" - ] - finding.save_no_options() - - skipped_risk_accept_count = 0 - if form.cleaned_data["risk_acceptance"]: - for finding in finds: - if not finding.duplicate: - if form.cleaned_data["risk_accept"]: - if ( - not finding.test.engagement.product.enable_simple_risk_acceptance - ): - skipped_risk_accept_count += 1 - else: - ra_helper.simple_risk_accept(request.user, finding) - elif form.cleaned_data["risk_unaccept"]: - ra_helper.risk_unaccept(request.user, finding) - - for prod in prods: - calculate_grade(prod) - - if skipped_risk_accept_count > 0: - messages.add_message( - request, - messages.WARNING, - (f"Skipped simple risk acceptance of {skipped_risk_accept_count} findings, " - "simple risk acceptance is disabled on the related products"), - extra_tags="alert-warning", - ) - - if form.cleaned_data["finding_group_create"]: - logger.debug("finding_group_create checked!") - finding_group_name = form.cleaned_data["finding_group_create_name"] - logger.debug("finding_group_create_name: %s", finding_group_name) - finding_group, added, skipped = finding_helper.create_finding_group( - finds, finding_group_name, - ) - - if added: - add_success_message_to_response( - f"Created finding group with {added} findings", - ) - return_url = reverse( - "view_finding_group", args=(finding_group.id,), - ) - - if skipped: - add_success_message_to_response( - f"Skipped {skipped} findings in group creation, findings already part of another group", - ) - - # refresh findings from db - finds = finds.all() - - if form.cleaned_data["finding_group_add"]: - logger.debug("finding_group_add checked!") - fgid = form.cleaned_data["add_to_finding_group_id"] - finding_group = Finding_Group.objects.get(id=fgid) - finding_group, added, skipped = finding_helper.add_to_finding_group( - finding_group, finds, - ) - - if added: - add_success_message_to_response( - f"Added {added} findings to finding group {finding_group.name}", - ) - return_url = reverse( - "view_finding_group", args=(finding_group.id,), - ) - - if skipped: - add_success_message_to_response( - f"Skipped {skipped} findings when adding to finding group {finding_group.name}, " - "findings already part of another group", - ) - - # refresh findings from db - finds = finds.all() + actually_updated_count = 0 - if form.cleaned_data["finding_group_remove"]: - logger.debug("finding_group_remove checked!") - ( - finding_groups, - removed, - skipped, - ) = finding_helper.remove_from_finding_group(finds) - - if removed: - add_success_message_to_response( - "Removed {} findings from finding groups {}".format( - removed, - ",".join( - [ - finding_group.name - for finding_group in finding_groups - ], - ), - ), - ) + _skipped_duplicate_count, actually_updated_count = _bulk_update_finding_status_and_severity( + finds, form, request, system_settings, prods, now, + ) - if skipped: - add_success_message_to_response( - f"Skipped {skipped} findings when removing from any finding group, findings not part of any group", - ) + _bulk_update_simple_fields(finds, form) - # refresh findings from db - finds = finds.all() - - if form.cleaned_data["finding_group_by"]: - logger.debug("finding_group_by checked!") - logger.debug(form.cleaned_data) - finding_group_by_option = form.cleaned_data[ - "finding_group_by_option" - ] - logger.debug("finding_group_by_option: %s", finding_group_by_option) - - ( - finding_groups, - grouped, - skipped, - groups_created, - ) = finding_helper.group_findings_by(finds, finding_group_by_option) - - if grouped: - add_success_message_to_response( - f"Grouped {grouped} findings into {len(finding_groups)} ({groups_created} newly created) finding groups", - ) - - if skipped: - add_success_message_to_response( - f"Skipped {skipped} findings when grouping by {finding_group_by_option} as these findings " - "were already in an existing group", - ) + _skipped_risk_accept_count, _skipped_active_risk_accept_count = _bulk_update_risk_acceptance( + finds, form, request, prods, + ) - # refresh findings from db - finds = finds.all() + group_return_url, finds = _bulk_update_finding_groups(finds, form) + if group_return_url: + return_url = group_return_url if form.cleaned_data["push_to_github"]: logger.debug("push selected findings to github") @@ -2844,96 +3111,25 @@ def finding_bulk_update_all(request, pid=None): # Delegate parsing and handling of strings/iterables to helper bulk_add_tags_to_instances(tag_or_tags=tags, instances=finds, tag_field_name="tags") - error_counts = defaultdict(lambda: 0) - success_count = 0 - finding_groups = set( # noqa: C401 - finding.finding_group - for finding in finds - if finding.has_finding_group - and ( - jira_helper.is_push_all_issues(finding) - or jira_helper.is_keep_in_sync_with_jira(finding) - or form.cleaned_data.get("push_to_jira") - ) - ) - logger.debug("finding_groups: %s", finding_groups) - for group in finding_groups: - if form.cleaned_data.get("push_to_jira"): - ( - can_be_pushed_to_jira, - error_message, - _error_code, - ) = jira_helper.can_be_pushed_to_jira(group) - if not can_be_pushed_to_jira: - error_counts[error_message] += 1 - jira_helper.log_jira_cannot_be_pushed_reason(error_message, group) - else: - logger.debug( - "pushing to jira from finding.finding_bulk_update_all()", - ) - jira_helper.push_to_jira(group) - success_count += 1 - - for error_message, error_count in error_counts.items(): - add_error_message_to_response(f"{error_count} finding groups could not be pushed to JIRA: {error_message}") - - if success_count > 0: - add_success_message_to_response(f"{success_count} finding groups pushed to JIRA successfully") - - # refresh from db - finds = finds.all() - - error_counts = defaultdict(lambda: 0) - success_count = 0 - for finding in finds: - tool_issue_updater.async_tool_issue_update(finding) - - # not sure yet if we want to support bulk unlink, so leave as commented out for now - # if form.cleaned_data['unlink_from_jira']: - # if finding.has_jira_issue: - # jira_helper.finding_unlink_jira(request, finding) - - # Because we never call finding.save() in a bulk update, we need to actually - # push the JIRA stuff here, rather than in finding.save() - # can't use helper as when push_all_jira_issues is True, - # the checkbox gets disabled and is always false - # push_to_jira = jira_helper.is_push_to_jira(new_finding, - # form.cleaned_data.get('push_to_jira')) - if ( - form.cleaned_data.get("push_to_jira") - or jira_helper.is_push_all_issues(finding) - or jira_helper.is_keep_in_sync_with_jira(finding) - ) and not finding.has_finding_group: - ( - can_be_pushed_to_jira, - error_message, - _error_code, - ) = jira_helper.can_be_pushed_to_jira(finding) - if finding.has_jira_group_issue and not finding.has_jira_issue: - error_message = ( - "finding already pushed as part of Finding Group" - ) - error_counts[error_message] += 1 - jira_helper.log_jira_cannot_be_pushed_reason(error_message, finding) - elif not can_be_pushed_to_jira: - error_counts[error_message] += 1 - jira_helper.log_jira_cannot_be_pushed_reason(error_message, finding) - else: - logger.debug( - "pushing to jira from finding.finding_bulk_update_all()", - ) - jira_helper.push_to_jira(finding) - if note is not None and isinstance(note, Notes): - jira_helper.add_comment(finding, note) - success_count += 1 - - for error_message, error_count in error_counts.items(): - add_error_message_to_response(f"{error_count} findings could not be pushed to JIRA: {error_message}") + _bulk_push_to_jira(finds, form, note) - if success_count > 0: - add_success_message_to_response(f"{success_count} findings pushed to JIRA successfully") - - if updated_find_count > 0: + # Show success message if status/severity updates were made (using actually_updated_count) + # or if other updates were made (using updated_find_count) + if (form.cleaned_data["severity"] or form.cleaned_data["status"]) and actually_updated_count > 0: + messages.add_message( + request, + messages.SUCCESS, + f"Bulk update of {actually_updated_count} findings was successful.", + extra_tags="alert-success", + ) + elif updated_find_count > 0 and ( + form.cleaned_data["date"] or form.cleaned_data["planned_remediation_date"] + or form.cleaned_data["planned_remediation_version"] or form.cleaned_data["tags"] + or form.cleaned_data["notes"] or form.cleaned_data["risk_acceptance"] + or form.cleaned_data["finding_group_create"] or form.cleaned_data["finding_group_add"] + or form.cleaned_data["finding_group_remove"] or form.cleaned_data["finding_group_by"] + or form.cleaned_data["push_to_jira"] or form.cleaned_data["push_to_github"] + ): messages.add_message( request, messages.SUCCESS, diff --git a/dojo/finding_group/signals.py b/dojo/finding_group/signals.py index 3e7ffe7c7b7..4b41b838983 100644 --- a/dojo/finding_group/signals.py +++ b/dojo/finding_group/signals.py @@ -1,9 +1,7 @@ import contextlib -from auditlog.models import LogEntry from django.conf import settings from django.contrib.auth import get_user_model -from django.contrib.contenttypes.models import ContentType from django.db.models.signals import post_delete from django.dispatch import receiver from django.urls import reverse @@ -21,7 +19,7 @@ def finding_group_post_delete(sender, instance, using, origin, **kwargs): user = None if settings.ENABLE_AUDITLOG: - # First try to find deletion author in pghistory events + # Find deletion author in pghistory events # Look for delete events for this specific finding_group instance pghistory_delete_events = DojoEvents.objects.filter( pgh_obj_model="dojo.Finding_Group", @@ -37,15 +35,6 @@ def finding_group_post_delete(sender, instance, using, origin, **kwargs): with contextlib.suppress(User.DoesNotExist): user = User.objects.get(id=latest_delete.user) - # Fall back to django-auditlog if no user found in pghistory - if not user: - if le := LogEntry.objects.filter( - action=LogEntry.Action.DELETE, - content_type=ContentType.objects.get(app_label="dojo", model="finding_group"), - object_id=instance.id, - ).order_by("-id").first(): - user = le.actor - # Update description with user if found if user: description = _('The finding group "%(name)s" was deleted by %(user)s') % { diff --git a/dojo/fixtures/defect_dojo_sample_data.json b/dojo/fixtures/defect_dojo_sample_data.json index 4dd8181df5f..a9dbc052b5d 100644 --- a/dojo/fixtures/defect_dojo_sample_data.json +++ b/dojo/fixtures/defect_dojo_sample_data.json @@ -822,7 +822,6 @@ "product_grade_f": 59, "enable_product_tag_inheritance": false, "enable_benchmark": true, - "enable_template_match": false, "enable_similar_findings": true, "engagement_auto_close": false, "engagement_auto_close_days": 3, @@ -32239,8 +32238,6 @@ "references": "", "last_used": null, "numerical_severity": null, - "template_match": false, - "template_match_title": false, "tags": [] } }, @@ -32259,8 +32256,6 @@ "references": "", "last_used": null, "numerical_severity": null, - "template_match": false, - "template_match_title": false, "tags": [] } }, @@ -32279,8 +32274,6 @@ "references": "", "last_used": null, "numerical_severity": null, - "template_match": false, - "template_match_title": false, "tags": [] } }, diff --git a/dojo/fixtures/test_type.json b/dojo/fixtures/test_type.json index d1a9fa60726..898560e9d98 100644 --- a/dojo/fixtures/test_type.json +++ b/dojo/fixtures/test_type.json @@ -47,5 +47,12 @@ }, "model": "dojo.test_type", "pk": 7 + }, + { + "fields": { + "name": "Prowler" + }, + "model": "dojo.test_type", + "pk": 8 } ] \ No newline at end of file diff --git a/dojo/forms.py b/dojo/forms.py index 636ce1be9c4..a73abb00ce6 100644 --- a/dojo/forms.py +++ b/dojo/forms.py @@ -31,7 +31,7 @@ from tagulous.forms import TagField import dojo.jira_link.helper as jira_helper -from dojo.authorization.authorization import user_has_configuration_permission +from dojo.authorization.authorization import user_has_configuration_permission, user_is_superuser_or_global_owner from dojo.authorization.roles_permissions import Permissions from dojo.endpoint.utils import endpoint_filter, endpoint_get_or_create, validate_endpoints_to_add from dojo.engagement.queries import get_authorized_engagements @@ -114,7 +114,7 @@ is_finding_groups_enabled, is_scan_file_too_large, ) -from dojo.validators import ImporterFileExtensionValidator, tag_validator +from dojo.validators import ImporterFileExtensionValidator, cvss3_validator, cvss4_validator, tag_validator from dojo.widgets import TableCheckboxWidget logger = logging.getLogger(__name__) @@ -1612,7 +1612,9 @@ class ApplyFindingTemplateForm(forms.Form): cwe = forms.IntegerField(label="CWE", required=False) vulnerability_ids = vulnerability_ids_field cvssv3 = forms.CharField(label="CVSSv3", max_length=117, required=False, widget=forms.TextInput(attrs={"class": "btn btn-secondary dropdown-toggle", "data-toggle": "dropdown", "aria-haspopup": "true", "aria-expanded": "false"})) - cvssv4 = forms.CharField(label="CVSSv3", max_length=117, required=False) + cvssv3_score = forms.FloatField(required=False, label="CVSSv3 Score") + cvssv4 = forms.CharField(label="CVSSv4", max_length=255, required=False) + cvssv4_score = forms.FloatField(required=False, label="CVSSv4 Score") severity = forms.ChoiceField(required=False, choices=SEVERITY_CHOICES, error_messages={"required": "Select valid choice: In Progress, On Hold, Completed", "invalid_choice": "Select valid choice: Critical,High,Medium,Low"}) @@ -1621,6 +1623,26 @@ class ApplyFindingTemplateForm(forms.Form): impact = forms.CharField(widget=forms.Textarea, required=False) references = forms.CharField(widget=forms.Textarea, required=False) + # Remediation planning fields + fix_available = forms.BooleanField(required=False) + fix_version = forms.CharField(max_length=100, required=False) + planned_remediation_version = forms.CharField(max_length=99, required=False) + effort_for_fixing = forms.CharField(max_length=99, required=False) + + # Technical details fields + steps_to_reproduce = forms.CharField(widget=forms.Textarea, required=False) + severity_justification = forms.CharField(widget=forms.Textarea, required=False) + component_name = forms.CharField(max_length=500, required=False) + component_version = forms.CharField(max_length=100, required=False) + + # Notes field + notes = forms.CharField(widget=forms.Textarea, required=False, help_text="Note content to add when applying template") + + # Endpoints field + endpoints = forms.CharField(max_length=5000, required=False, + help_text="Endpoint URLs (one per line)", + widget=forms.widgets.Textarea(attrs={"rows": "3", "cols": "400"})) + tags = TagField(required=False, help_text="Add tags that help describe this finding template. Choose from the list or add new tags. Press Enter key to add.", initial=Finding.tags.tag_model.objects.all().order_by("name")) def __init__(self, template=None, *args, **kwargs): @@ -1628,7 +1650,36 @@ def __init__(self, template=None, *args, **kwargs): self.fields["tags"].autocomplete_tags = Finding.tags.tag_model.objects.all().order_by("name") self.template = template if template: - self.template.vulnerability_ids = "\n".join(template.vulnerability_ids) + # Populate vulnerability_ids field initial value + self.fields["vulnerability_ids"].initial = "\n".join(template.vulnerability_ids) + + # Populate CVSS fields from template + if hasattr(template, "cvssv3"): + self.fields["cvssv3"].initial = template.cvssv3 + if hasattr(template, "cvssv4"): + self.fields["cvssv4"].initial = template.cvssv4 + if hasattr(template, "cvssv3_score"): + self.fields["cvssv3_score"].initial = template.cvssv3_score + if hasattr(template, "cvssv4_score"): + self.fields["cvssv4_score"].initial = template.cvssv4_score + + # Populate all other new fields from template + for field_name in ["fix_available", "fix_version", "planned_remediation_version", + "effort_for_fixing", "steps_to_reproduce", "severity_justification", + "component_name", "component_version", "notes"]: + if hasattr(template, field_name): + value = getattr(template, field_name) + if value is not None: + self.fields[field_name].initial = value + + # Populate endpoints + if hasattr(template, "endpoints"): + endpoints_value = template.endpoints + if endpoints_value: + if isinstance(endpoints_value, list): + self.fields["endpoints"].initial = "\n".join(endpoints_value) + else: + self.fields["endpoints"].initial = endpoints_value # Hide CVSS fields based on system settings hide_cvss_fields_if_disabled(self) @@ -1651,17 +1702,27 @@ def clean_tags(self): return self.cleaned_data.get("tags") class Meta: - fields = ["title", "cwe", "vulnerability_ids", "cvssv3", "cvssv4", "severity", "description", "mitigation", "impact", "references", "tags"] - order = ("title", "cwe", "vulnerability_ids", "cvssv3", "cvssv4", "severity", "description", "impact", "is_mitigated") + fields = ["title", "cwe", "vulnerability_ids", "cvssv3", "cvssv3_score", "cvssv4", "cvssv4_score", + "severity", "description", "mitigation", "impact", "references", "tags", + "fix_available", "fix_version", "planned_remediation_version", "effort_for_fixing", + "steps_to_reproduce", "severity_justification", "component_name", "component_version", + "notes", "endpoints"] + order = ("title", "cwe", "vulnerability_ids", "cvssv3", "cvssv3_score", "cvssv4", "cvssv4_score", + "severity", "description", "impact", "steps_to_reproduce", "severity_justification", + "mitigation", "fix_available", "fix_version", "planned_remediation_version", + "effort_for_fixing", "component_name", "component_version", "references", "notes", + "endpoints", "tags") class FindingTemplateForm(forms.ModelForm): - apply_to_findings = forms.BooleanField(required=False, help_text="Apply template to all findings that match this CWE. (Update will overwrite mitigation, impact and references for any active, verified findings.)") title = forms.CharField(max_length=1000, required=True) cwe = forms.IntegerField(label="CWE", required=False) vulnerability_ids = vulnerability_ids_field cvssv3 = forms.CharField(label="CVSS3 Vector", max_length=117, required=False, widget=forms.TextInput(attrs={"class": "btn btn-secondary dropdown-toggle", "data-toggle": "dropdown", "aria-haspopup": "true", "aria-expanded": "false"})) + cvssv3_score = forms.FloatField(required=False, label="CVSSv3 Score") + cvssv4 = forms.CharField(label="CVSS4 Vector", max_length=255, required=False) + cvssv4_score = forms.FloatField(required=False, label="CVSSv4 Score") severity = forms.ChoiceField( required=False, choices=SEVERITY_CHOICES, @@ -1669,7 +1730,30 @@ class FindingTemplateForm(forms.ModelForm): "required": "Select valid choice: In Progress, On Hold, Completed", "invalid_choice": "Select valid choice: Critical,High,Medium,Low"}) - field_order = ["title", "cwe", "vulnerability_ids", "severity", "cvssv3", "description", "mitigation", "impact", "references", "tags", "template_match", "template_match_cwe", "template_match_title", "apply_to_findings"] + # Remediation planning fields + fix_available = forms.BooleanField(required=False) + fix_version = forms.CharField(max_length=100, required=False) + planned_remediation_version = forms.CharField(max_length=99, required=False) + effort_for_fixing = forms.CharField(max_length=99, required=False) + + # Technical details fields + steps_to_reproduce = forms.CharField(widget=forms.Textarea, required=False) + severity_justification = forms.CharField(widget=forms.Textarea, required=False) + component_name = forms.CharField(max_length=500, required=False) + component_version = forms.CharField(max_length=100, required=False) + + # Notes field + notes = forms.CharField(widget=forms.Textarea, required=False, help_text="Note content to add when applying template") + + # Endpoints field + endpoints = forms.CharField(max_length=5000, required=False, + help_text="Endpoint URLs (one per line)", + widget=forms.widgets.Textarea(attrs={"rows": "3", "cols": "400"})) + + field_order = ["title", "cwe", "vulnerability_ids", "severity", "cvssv3", "cvssv3_score", "cvssv4", "cvssv4_score", + "description", "impact", "steps_to_reproduce", "severity_justification", "mitigation", + "fix_available", "fix_version", "planned_remediation_version", "effort_for_fixing", + "component_name", "component_version", "references", "notes", "endpoints", "tags"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -1680,8 +1764,29 @@ def __init__(self, *args, **kwargs): class Meta: model = Finding_Template - order = ("title", "cwe", "vulnerability_ids", "cvssv3", "severity", "description", "impact") - exclude = ("numerical_severity", "is_mitigated", "last_used", "endpoint_status", "cve") + order = ("title", "cwe", "vulnerability_ids", "cvssv3", "cvssv3_score", "cvssv4", "cvssv4_score", "severity", "description", "impact", + "steps_to_reproduce", "severity_justification", "mitigation", "fix_available", "fix_version", + "planned_remediation_version", "effort_for_fixing", "component_name", "component_version", + "references", "notes", "endpoints", "tags") + exclude = ("numerical_severity", "is_mitigated", "last_used", "endpoint_status", "cve", "vulnerability_ids_text") + + def clean_cvssv3(self): + value = self.cleaned_data.get("cvssv3") + if value: + try: + cvss3_validator(value) + except ValidationError as e: + raise forms.ValidationError(e.messages) + return value + + def clean_cvssv4(self): + value = self.cleaned_data.get("cvssv4") + if value: + try: + cvss4_validator(value) + except ValidationError as e: + raise forms.ValidationError(e.messages) + return value def clean_tags(self): tag_validator(self.cleaned_data.get("tags")) @@ -1738,6 +1843,9 @@ def clean(self): if cleaned_data["false_p"] and cleaned_data["verified"]: msg = "False positive findings cannot be verified." raise forms.ValidationError(msg) + if cleaned_data["active"] and cleaned_data.get("risk_acceptance") and cleaned_data.get("risk_accept"): + msg = "Active findings cannot be risk accepted." + raise forms.ValidationError(msg) return cleaned_data def clean_tags(self): @@ -2420,13 +2528,32 @@ class Meta: class UserContactInfoForm(forms.ModelForm): + reset_api_token = forms.BooleanField( + required=False, + label=_("Reset API token"), + help_text=_("Upon saving, a new token will be generated and a notification of category 'Other' is triggered."), + ) + class Meta: model = UserContactInfo exclude = ["user", "slack_user_id"] + # Swap order: password_last_reset before token_last_reset + field_order = [ + "title", "phone_number", "cell_number", "twitter_username", "github_username", + "slack_username", "block_execution", "force_password_reset", "reset_api_token", + "password_last_reset", "token_last_reset", + ] def __init__(self, *args, **kwargs): user = kwargs.pop("user", None) super().__init__(*args, **kwargs) + # Make timestamp fields readonly. + # NOTE: `disabled=True` is enforced server-side by Django forms: posted values for disabled fields + # are ignored during binding/cleaning, so these timestamps cannot be modified via this form. + if "password_last_reset" in self.fields: + self.fields["password_last_reset"].disabled = True + if "token_last_reset" in self.fields: + self.fields["token_last_reset"].disabled = True # Do not expose force password reset if the current user does not have a password to reset if user is not None: if not user.has_usable_password(): @@ -2437,11 +2564,15 @@ def __init__(self, *args, **kwargs): if not current_user.is_superuser: if not user_has_configuration_permission(current_user, "auth.change_user") and \ not user_has_configuration_permission(current_user, "auth.add_user"): - del self.fields["force_password_reset"] + self.fields.pop("force_password_reset", None) if not get_system_setting("enable_user_profile_editable"): for field in self.fields: self.fields[field].disabled = True + # Only show reset_api_token to superusers or global owners, and only if API tokens are enabled + if not settings.API_TOKENS_ENABLED or not user_is_superuser_or_global_owner(current_user): + self.fields.pop("reset_api_token", None) + class GlobalRoleForm(forms.ModelForm): class Meta: diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index 380fa24e4e0..b9a0289f9ef 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -1,6 +1,7 @@ import base64 import logging import time +from collections.abc import Iterable from celery import chord, group from django.conf import settings @@ -32,7 +33,6 @@ Test_Import, Test_Import_Finding_Action, Test_Type, - Vulnerability_Id, ) from dojo.notifications.helper import create_notification from dojo.tag_utils import bulk_add_tags_to_instances @@ -302,6 +302,7 @@ def determine_process_method( def determine_deduplication_algorithm(self) -> str: """ Determines what dedupe algorithm to use for the Test being processed. + Overridden in Pro. :return: A string representing the dedupe algorithm to use. """ return self.test.deduplication_algorithm @@ -359,6 +360,71 @@ def update_test_tags(self): if self.tags is not None and len(self.tags) > 0: self.test.tags.set(self.tags) + def apply_import_tags( + self, + new_findings: Iterable[Finding] | None = None, + closed_findings: Iterable[Finding] | None = None, + reactivated_findings: Iterable[Finding] | None = None, + untouched_findings: Iterable[Finding] | None = None, + ) -> None: + """Apply tags to findings and endpoints from an import operation.""" + # Normalize None values to empty lists and convert sets/other iterables to lists + if untouched_findings is None: + untouched_findings = [] + elif not isinstance(untouched_findings, list): + untouched_findings = list(untouched_findings) + + if reactivated_findings is None: + reactivated_findings = [] + elif not isinstance(reactivated_findings, list): + reactivated_findings = list(reactivated_findings) + + if closed_findings is None: + closed_findings = [] + elif not isinstance(closed_findings, list): + closed_findings = list(closed_findings) + + if new_findings is None: + new_findings = [] + elif not isinstance(new_findings, list): + new_findings = list(new_findings) + + # Collect all affected findings + findings_to_tag = new_findings + closed_findings + reactivated_findings + untouched_findings + + if not findings_to_tag: + return + + # Add any tags to the findings imported if necessary + if self.apply_tags_to_findings and self.tags: + findings_qs = Finding.objects.filter(id__in=[f.id for f in findings_to_tag]) + try: + bulk_add_tags_to_instances( + tag_or_tags=self.tags, + instances=findings_qs, + tag_field_name="tags", + ) + except IntegrityError: + # Fallback to safe per-instance tagging if concurrent deletes occur + for finding in findings_to_tag: + for tag in self.tags: + self.add_tags_safe(finding, tag) + + # Add any tags to any endpoints of the findings imported if necessary + if self.apply_tags_to_endpoints and self.tags: + endpoints_qs = Endpoint.objects.filter(finding__in=findings_to_tag).distinct() + try: + bulk_add_tags_to_instances( + tag_or_tags=self.tags, + instances=endpoints_qs, + tag_field_name="tags", + ) + except IntegrityError: + for finding in findings_to_tag: + for endpoint in finding.endpoints.all(): + for tag in self.tags: + self.add_tags_safe(endpoint, tag) + def update_import_history( self, new_findings: list[Finding] | None = None, @@ -379,6 +445,7 @@ def update_import_history( closed_findings = [] if new_findings is None: new_findings = [] + # Log the current state of what has occurred in case there could be # deviation from what is displayed in the view logger.debug( @@ -420,8 +487,8 @@ def update_import_history( # In longer running imports it can happen that the async_dupe_delete task removes a finding before the history record is created # We filter out these findings here to avoid FK violations (IntegrityError) all_findings = [] - for _list, _ in finding_action_mappings: - all_findings.extend(_list) + for list_, _ in finding_action_mappings: + all_findings.extend(list_) existing_findings = finding_helper.filter_findings_by_existence(all_findings) if all_findings else [] existing_ids = {f.id for f in existing_findings} @@ -454,37 +521,6 @@ def update_import_history( for record in import_history_records: self.create_import_history_record_safe(record) - # Add any tags to the findings imported if necessary - if self.apply_tags_to_findings and self.tags: - findings_qs = test_import.findings_affected.all() - try: - bulk_add_tags_to_instances( - tag_or_tags=self.tags, - instances=findings_qs, - tag_field_name="tags", - ) - except IntegrityError: - # Fallback to safe per-instance tagging if concurrent deletes occur - for finding in findings_qs: - for tag in self.tags: - self.add_tags_safe(finding, tag) - - # Add any tags to any endpoints of the findings imported if necessary - if self.apply_tags_to_endpoints and self.tags: - # Collect all endpoints linked to the affected findings - endpoints_qs = Endpoint.objects.filter(finding__in=test_import.findings_affected.all()).distinct() - try: - bulk_add_tags_to_instances( - tag_or_tags=self.tags, - instances=endpoints_qs, - tag_field_name="tags", - ) - except IntegrityError: - for finding in test_import.findings_affected.all(): - for endpoint in finding.endpoints.all(): - for tag in self.tags: - self.add_tags_safe(endpoint, tag) - return test_import def create_import_history_record_safe( @@ -797,6 +833,11 @@ def process_endpoints( logger.debug("endpoints_to_add: %s", endpoints_to_add) self.endpoint_manager.chunk_endpoints_and_disperse(finding, endpoints_to_add) + def sanitize_vulnerability_ids(self, finding) -> None: + """Remove undisired vulnerability id values""" + if finding.unsaved_vulnerability_ids: + finding.unsaved_vulnerability_ids = [x for x in finding.unsaved_vulnerability_ids if x.strip()] + def process_cve( self, finding: Finding, @@ -805,6 +846,8 @@ def process_cve( # Synchronize the cve field with the unsaved_vulnerability_ids # We do this to be as flexible as possible to handle the fields until # the cve field is not needed anymore and can be removed. + # Remove undisired vulnerability ids + self.sanitize_vulnerability_ids(finding) if finding.unsaved_vulnerability_ids and finding.cve: # Make sure the first entry of the list is the value of the cve field finding.unsaved_vulnerability_ids.insert(0, finding.cve) @@ -817,21 +860,24 @@ def process_cve( return finding - def process_vulnerability_ids( + def store_vulnerability_ids( self, finding: Finding, ) -> Finding: """ - Parse the `unsaved_vulnerability_ids` field from findings after they are parsed - to create `Vulnerability_Id` objects with the finding associated correctly - """ - if finding.unsaved_vulnerability_ids: - # Remove old vulnerability ids - keeping this call only because of flake8 - Vulnerability_Id.objects.filter(finding=finding).delete() + Store vulnerability IDs for a finding. + Reads from finding.unsaved_vulnerability_ids and saves them overwriting existing ones. - # user the helper function - finding_helper.save_vulnerability_ids(finding, finding.unsaved_vulnerability_ids) + Args: + finding: The finding to store vulnerability IDs for + Returns: + The finding object + + """ + self.sanitize_vulnerability_ids(finding) + vulnerability_ids_to_process = finding.unsaved_vulnerability_ids or [] + finding_helper.save_vulnerability_ids(finding, vulnerability_ids_to_process, delete_existing=False) return finding def process_files( diff --git a/dojo/importers/default_importer.py b/dojo/importers/default_importer.py index 5a14728043f..35fe6712387 100644 --- a/dojo/importers/default_importer.py +++ b/dojo/importers/default_importer.py @@ -129,6 +129,11 @@ def process_scan( new_findings=new_findings, closed_findings=closed_findings, ) + # Apply tags to findings and endpoints + self.apply_import_tags( + new_findings=new_findings, + closed_findings=closed_findings, + ) # Send out some notifications to the user logger.debug("IMPORT_SCAN: Generating notifications") create_notification( @@ -234,7 +239,7 @@ def process_findings( # Process any files self.process_files(finding) # Process vulnerability IDs - finding = self.process_vulnerability_ids(finding) + finding = self.store_vulnerability_ids(finding) # Categorize this finding as a new one new_findings.append(finding) # all data is already saved on the finding, we only need to trigger post processing in batches diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index 10b3ac7148a..47ce8c61acd 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -1,5 +1,6 @@ import logging +from django.conf import settings from django.core.files.uploadedfile import TemporaryUploadedFile from django.core.serializers import serialize from django.db.models.query_utils import Q @@ -7,6 +8,12 @@ import dojo.finding.helper as finding_helper import dojo.jira_link.helper as jira_helper from dojo.decorators import we_want_async +from dojo.finding.deduplication import ( + find_candidates_for_deduplication_hash, + find_candidates_for_deduplication_uid_or_hash, + find_candidates_for_deduplication_unique_id, + find_candidates_for_reimport_legacy, +) from dojo.importers.base_importer import BaseImporter, Parser from dojo.importers.options import ImporterOptions from dojo.models import ( @@ -125,6 +132,13 @@ def process_scan( reactivated_findings=reactivated_findings, untouched_findings=untouched_findings, ) + # Apply tags to findings and endpoints + self.apply_import_tags( + new_findings=new_findings, + closed_findings=closed_findings, + reactivated_findings=reactivated_findings, + untouched_findings=untouched_findings, + ) # Send out som notifications to the user logger.debug("REIMPORT_SCAN: Generating notifications") updated_count = ( @@ -152,6 +166,93 @@ def process_scan( test_import_history, ) + def get_reimport_match_candidates_for_batch( + self, + batch_findings: list[Finding], + ) -> tuple[dict, dict, dict]: + """ + Fetch candidate matches for a batch of *unsaved* findings during reimport. + + This is intentionally a separate method so downstream editions (e.g. Dojo Pro) + can override candidate retrieval without copying the full `process_findings()` + implementation. + + Is overridden in Pro. + + Returns: + (candidates_by_hash, candidates_by_uid, candidates_by_key) + + """ + candidates_by_hash: dict = {} + candidates_by_uid: dict = {} + candidates_by_key: dict = {} + + if self.deduplication_algorithm == "hash_code": + candidates_by_hash = find_candidates_for_deduplication_hash( + self.test, + batch_findings, + mode="reimport", + ) + elif self.deduplication_algorithm == "unique_id_from_tool": + candidates_by_uid = find_candidates_for_deduplication_unique_id( + self.test, + batch_findings, + mode="reimport", + ) + elif self.deduplication_algorithm == "unique_id_from_tool_or_hash_code": + candidates_by_uid, candidates_by_hash = find_candidates_for_deduplication_uid_or_hash( + self.test, + batch_findings, + mode="reimport", + ) + elif self.deduplication_algorithm == "legacy": + candidates_by_key = find_candidates_for_reimport_legacy(self.test, batch_findings) + + return candidates_by_hash, candidates_by_uid, candidates_by_key + + def add_new_finding_to_candidates( + self, + finding: Finding, + candidates_by_hash: dict, + candidates_by_uid: dict, + candidates_by_key: dict, + ) -> None: + """ + Add a newly created finding to candidate dictionaries for subsequent findings in the same batch. + + This allows duplicates within the same scan report to be detected even when they're processed + in the same batch. When a new finding is created (no match found), it is added to the candidate + dictionaries so that subsequent findings in the same batch can match against it. + + Is overriden in Pro + + Args: + finding: The newly created finding to add to candidates + candidates_by_hash: Dictionary mapping hash_code to list of findings (modified in-place) + candidates_by_uid: Dictionary mapping unique_id_from_tool to list of findings (modified in-place) + candidates_by_key: Dictionary mapping (title_lower, severity) to list of findings (modified in-place) + + """ + if not finding: + return + + if finding.hash_code: + candidates_by_hash.setdefault(finding.hash_code, []).append(finding) + deduplicationLogger.debug( + f"Added finding {finding.id} (hash_code: {finding.hash_code}) to candidates for next findings in this report", + ) + if finding.unique_id_from_tool: + candidates_by_uid.setdefault(finding.unique_id_from_tool, []).append(finding) + deduplicationLogger.debug( + f"Added finding {finding.id} (unique_id_from_tool: {finding.unique_id_from_tool}) to candidates for next findings in this report", + ) + if finding.title: + legacy_key = (finding.title.lower(), finding.severity) + candidates_by_key.setdefault(legacy_key, []).append(finding) + deduplicationLogger.debug( + f"Added finding {finding.id} (title: {finding.title}, severity: {finding.severity}) to candidates for next findings in this report", + ) + def process_findings( self, parsed_findings: list[Finding], @@ -182,8 +283,6 @@ def process_findings( self.reactivated_items = [] self.unchanged_items = [] self.group_names_to_findings_dict = {} - # Progressive batching for chord execution - # No chord: we dispatch per 1000 findings or on the final finding logger.debug(f"starting reimport of {len(parsed_findings) if parsed_findings else 0} items.") logger.debug("STEP 1: looping over findings from the reimported report and trying to match them to existing findings") @@ -204,86 +303,136 @@ def process_findings( cleaned_findings.append(sanitized) batch_finding_ids: list[int] = [] - batch_max_size = 1000 - - for idx, unsaved_finding in enumerate(cleaned_findings): - is_final = idx == len(cleaned_findings) - 1 - # Some parsers provide "mitigated" field but do not set timezone (because they are probably not available in the report) - # Finding.mitigated is DateTimeField and it requires timezone - if unsaved_finding.mitigated and not unsaved_finding.mitigated.tzinfo: - unsaved_finding.mitigated = unsaved_finding.mitigated.replace(tzinfo=self.now.tzinfo) - # Override the test if needed - if not hasattr(unsaved_finding, "test"): - unsaved_finding.test = self.test - # Set the service supplied at import time - if self.service is not None: - unsaved_finding.service = self.service - # Clean any endpoints that are on the finding - self.endpoint_manager.clean_unsaved_endpoints(unsaved_finding.unsaved_endpoints) - # Calculate the hash code to be used to identify duplicates - unsaved_finding.hash_code = self.calculate_unsaved_finding_hash_code(unsaved_finding) - deduplicationLogger.debug(f"unsaved finding's hash_code: {unsaved_finding.hash_code}") - # Match any findings to this new one coming in - matched_findings = self.match_new_finding_to_existing_finding(unsaved_finding) - deduplicationLogger.debug(f"found {len(matched_findings)} findings matching with current new finding") - # Determine how to proceed based on whether matches were found or not - if matched_findings: - existing_finding = matched_findings[0] - finding, force_continue = self.process_matched_finding( + # Batch size for deduplication/post-processing (only new findings) + dedupe_batch_max_size = getattr(settings, "IMPORT_REIMPORT_DEDUPE_BATCH_SIZE", 1000) + # Batch size for candidate matching (all findings, before matching) + match_batch_max_size = getattr(settings, "IMPORT_REIMPORT_MATCH_BATCH_SIZE", 1000) + + # Process findings in batches to enable batch candidate fetching + # This avoids the 1+N query problem by fetching all candidates for a batch at once + for batch_start in range(0, len(cleaned_findings), match_batch_max_size): + batch_end = min(batch_start + match_batch_max_size, len(cleaned_findings)) + batch_findings = cleaned_findings[batch_start:batch_end] + is_final_batch = batch_end == len(cleaned_findings) + + logger.debug(f"Processing reimport batch {batch_start}-{batch_end} of {len(cleaned_findings)} findings") + + # Prepare findings in batch: set test, service, calculate hash codes + for unsaved_finding in batch_findings: + # Some parsers provide "mitigated" field but do not set timezone (because they are probably not available in the report) + # Finding.mitigated is DateTimeField and it requires timezone + if unsaved_finding.mitigated and not unsaved_finding.mitigated.tzinfo: + unsaved_finding.mitigated = unsaved_finding.mitigated.replace(tzinfo=self.now.tzinfo) + # Override the test if needed + if not hasattr(unsaved_finding, "test"): + unsaved_finding.test = self.test + # Set the service supplied at import time + if self.service is not None: + unsaved_finding.service = self.service + # Clean any endpoints that are on the finding + self.endpoint_manager.clean_unsaved_endpoints(unsaved_finding.unsaved_endpoints) + # Calculate the hash code to be used to identify duplicates + unsaved_finding.hash_code = self.calculate_unsaved_finding_hash_code(unsaved_finding) + deduplicationLogger.debug(f"unsaved finding's hash_code: {unsaved_finding.hash_code}") + + # Fetch all candidates for this batch at once (batch candidate finding) + candidates_by_hash, candidates_by_uid, candidates_by_key = self.get_reimport_match_candidates_for_batch( + batch_findings, + ) + + # Process each finding in the batch using pre-fetched candidates + for idx, unsaved_finding in enumerate(batch_findings): + is_final = is_final_batch and idx == len(batch_findings) - 1 + + # Match any findings to this new one coming in using pre-fetched candidates + matched_findings = self.match_finding_to_candidate_reimport( unsaved_finding, - existing_finding, + candidates_by_hash=candidates_by_hash, + candidates_by_uid=candidates_by_uid, + candidates_by_key=candidates_by_key, ) - # Determine if we should skip the rest of the loop - if force_continue: - continue - # Update endpoints on the existing finding with those on the new finding - if finding.dynamic_finding: - logger.debug( - "Re-import found an existing dynamic finding for this new " - "finding. Checking the status of endpoints", - ) - self.endpoint_manager.update_endpoint_status( - existing_finding, + deduplicationLogger.debug(f"found {len(matched_findings)} findings matching with current new finding") + # Determine how to proceed based on whether matches were found or not + if matched_findings: + existing_finding = matched_findings[0] + finding, force_continue = self.process_matched_finding( unsaved_finding, - self.user, + existing_finding, ) - else: - finding = self.process_finding_that_was_not_matched(unsaved_finding) - # This condition __appears__ to always be true, but am afraid to remove it - if finding: - # Process the rest of the items on the finding - finding = self.finding_post_processing( - finding, - unsaved_finding, - ) - # all data is already saved on the finding, we only need to trigger post processing in batches - push_to_jira = self.push_to_jira and (not self.findings_groups_enabled or not self.group_by) - batch_finding_ids.append(finding.id) - - # If batch is full or we're at the end, dispatch one batched task - if len(batch_finding_ids) >= batch_max_size or is_final: - finding_ids_batch = list(batch_finding_ids) - batch_finding_ids.clear() - if we_want_async(async_user=self.user): - finding_helper.post_process_findings_batch_signature( - finding_ids_batch, - dedupe_option=True, - rules_option=True, - product_grading_option=True, - issue_updater_option=True, - push_to_jira=push_to_jira, - )() - else: - finding_helper.post_process_findings_batch( - finding_ids_batch, - dedupe_option=True, - rules_option=True, - product_grading_option=True, - issue_updater_option=True, - push_to_jira=push_to_jira, + # Determine if we should skip the rest of the loop + if force_continue: + continue + # Update endpoints on the existing finding with those on the new finding + if finding.dynamic_finding: + logger.debug( + "Re-import found an existing dynamic finding for this new " + "finding. Checking the status of endpoints", + ) + self.endpoint_manager.update_endpoint_status( + existing_finding, + unsaved_finding, + self.user, ) + else: + finding = self.process_finding_that_was_not_matched(unsaved_finding) + + # Add newly created finding to candidates for subsequent findings in this batch + self.add_new_finding_to_candidates( + finding, + candidates_by_hash, + candidates_by_uid, + candidates_by_key, + ) - # No chord: tasks are dispatched immediately above per batch + # This condition __appears__ to always be true, but am afraid to remove it + if finding: + # Process the rest of the items on the finding + finding = self.finding_post_processing( + finding, + unsaved_finding, + ) + # all data is already saved on the finding, we only need to trigger post processing in batches + push_to_jira = self.push_to_jira and (not self.findings_groups_enabled or not self.group_by) + batch_finding_ids.append(finding.id) + + # Post-processing batches (deduplication, rules, etc.) are separate from matching batches. + # These batches only contain "new" findings that were saved (not matched to existing findings). + # In reimport scenarios, typically most findings match existing ones, so only a small fraction + # of findings in each matching batch become new findings that need deduplication. + # + # We accumulate finding IDs across matching batches rather than dispatching at the end of each + # matching batch. This ensures deduplication batches stay close to the intended batch size + # (e.g., 1000 findings) for optimal bulk operation efficiency, even when only ~10% of findings + # in matching batches are new. If we dispatched at the end of each matching batch, we would + # end up with many small deduplication batches (e.g., ~100 findings each), reducing efficiency. + # + # The two batch types serve different purposes: + # - Matching batches: optimize candidate fetching (solve 1+N query problem) + # - Deduplication batches: optimize bulk operations (larger batches = fewer queries) + # They don't need to be aligned since they optimize different operations. + if len(batch_finding_ids) >= dedupe_batch_max_size or is_final: + finding_ids_batch = list(batch_finding_ids) + batch_finding_ids.clear() + if we_want_async(async_user=self.user): + finding_helper.post_process_findings_batch_signature( + finding_ids_batch, + dedupe_option=True, + rules_option=True, + product_grading_option=True, + issue_updater_option=True, + push_to_jira=push_to_jira, + )() + else: + finding_helper.post_process_findings_batch( + finding_ids_batch, + dedupe_option=True, + rules_option=True, + product_grading_option=True, + issue_updater_option=True, + push_to_jira=push_to_jira, + ) + + # No chord: tasks are dispatched immediately above per batch self.to_mitigate = (set(self.original_items) - set(self.reactivated_items) - set(self.unchanged_items)) # due to #3958 we can have duplicates inside the same report @@ -375,48 +524,73 @@ def parse_findings_dynamic_test_type( logger.debug("REIMPORT_SCAN parser v2: Create parse findings") return super().parse_findings_dynamic_test_type(scan, parser) - def match_new_finding_to_existing_finding( + def match_finding_to_candidate_reimport( self, unsaved_finding: Finding, + candidates_by_hash: dict | None = None, + candidates_by_uid: dict | None = None, + candidates_by_key: dict | None = None, ) -> list[Finding]: - """Matches a single new finding to N existing findings and then returns those matches""" - # This code should match the logic used for deduplication out of the re-import feature. - # See utils.py deduplicate_* functions - deduplicationLogger.debug("return findings bases on algorithm: %s", self.deduplication_algorithm) + """ + Matches a single new finding to existing findings using pre-fetched candidate dictionaries. + This avoids individual database queries by using batch-fetched candidates. + + Args: + unsaved_finding: The finding to match + candidates_by_hash: Dictionary mapping hash_code to list of findings (for hash_code algorithm) + candidates_by_uid: Dictionary mapping unique_id_from_tool to list of findings (for unique_id algorithms) + candidates_by_key: Dictionary mapping (title_lower, severity) to list of findings (for legacy algorithm) + + Returns: + List of matching findings, ordered by id + + """ + deduplicationLogger.debug("matching finding for reimport using algorithm: %s", self.deduplication_algorithm) + if self.deduplication_algorithm == "hash_code": - return Finding.objects.filter( - test=self.test, - hash_code=unsaved_finding.hash_code, - ).exclude(hash_code=None).order_by("id") + if candidates_by_hash is None or unsaved_finding.hash_code is None: + return [] + matches = candidates_by_hash.get(unsaved_finding.hash_code, []) + return sorted(matches, key=lambda f: f.id) + if self.deduplication_algorithm == "unique_id_from_tool": - deduplicationLogger.debug(f"unique_id_from_tool: {unsaved_finding.unique_id_from_tool}") - return Finding.objects.filter( - test=self.test, - unique_id_from_tool=unsaved_finding.unique_id_from_tool, - ).exclude(unique_id_from_tool=None).order_by("id") + if candidates_by_uid is None or unsaved_finding.unique_id_from_tool is None: + return [] + matches = candidates_by_uid.get(unsaved_finding.unique_id_from_tool, []) + return sorted(matches, key=lambda f: f.id) + if self.deduplication_algorithm == "unique_id_from_tool_or_hash_code": - deduplicationLogger.debug(f"unique_id_from_tool: {unsaved_finding.unique_id_from_tool}") - deduplicationLogger.debug(f"hash_code: {unsaved_finding.hash_code}") - query = Finding.objects.filter( - Q(test=self.test), - (Q(hash_code__isnull=False) & Q(hash_code=unsaved_finding.hash_code)) - | (Q(unique_id_from_tool__isnull=False) & Q(unique_id_from_tool=unsaved_finding.unique_id_from_tool)), - ).order_by("id") - deduplicationLogger.debug(query.query) - return query + if candidates_by_hash is None and candidates_by_uid is None: + return [] + + if unsaved_finding.hash_code is None and unsaved_finding.unique_id_from_tool is None: + return [] + + # Collect matches from both hash_code and unique_id_from_tool + matches_by_id = {} + + if unsaved_finding.hash_code is not None: + hash_matches = candidates_by_hash.get(unsaved_finding.hash_code, []) + for match in hash_matches: + matches_by_id[match.id] = match + + if unsaved_finding.unique_id_from_tool is not None: + uid_matches = candidates_by_uid.get(unsaved_finding.unique_id_from_tool, []) + for match in uid_matches: + matches_by_id[match.id] = match + + matches = list(matches_by_id.values()) + return sorted(matches, key=lambda f: f.id) + if self.deduplication_algorithm == "legacy": - # This is the legacy reimport behavior. Although it's pretty flawed and doesn't match the legacy algorithm for deduplication, - # this is left as is for simplicity. - # Re-writing the legacy deduplication here would be complicated and counter-productive. - # If you have use cases going through this section, you're advised to create a deduplication configuration for your parser - logger.warning("Legacy reimport. In case of issue, you're advised to create a deduplication configuration in order not to go through this section") - return Finding.objects.filter( - title__iexact=unsaved_finding.title, - test=self.test, - severity=unsaved_finding.severity, - numerical_severity=Finding.get_numerical_severity(unsaved_finding.severity)).order_by("id") + if candidates_by_key is None or not unsaved_finding.title: + return [] + key = (unsaved_finding.title.lower(), unsaved_finding.severity) + matches = candidates_by_key.get(key, []) + return sorted(matches, key=lambda f: f.id) + logger.error(f'Internal error: unexpected deduplication_algorithm: "{self.deduplication_algorithm}"') - return None + return [] def process_matched_finding( self, @@ -691,11 +865,46 @@ def process_finding_that_was_not_matched( self.process_request_response_pairs(unsaved_finding) return unsaved_finding + def reconcile_vulnerability_ids( + self, + finding: Finding, + ) -> Finding: + """ + Reconcile vulnerability IDs for an existing finding. + Checks if IDs have changed before updating to avoid unnecessary database operations. + Uses prefetched data if available, otherwise fetches efficiently. + + Args: + finding: The existing finding to reconcile vulnerability IDs for. + Must have unsaved_vulnerability_ids set. + + Returns: + The finding object + + """ + vulnerability_ids_to_process = finding.unsaved_vulnerability_ids or [] + + # Use prefetched data directly without triggering queries + existing_vuln_ids = {v.vulnerability_id for v in finding.vulnerability_id_set.all()} + new_vuln_ids = set(vulnerability_ids_to_process) + + # Early exit if unchanged + if existing_vuln_ids == new_vuln_ids: + logger.debug( + f"Skipping vulnerability_ids update for finding {finding.id} - " + f"vulnerability_ids unchanged: {sorted(existing_vuln_ids)}", + ) + return finding + + # Update if changed + finding_helper.save_vulnerability_ids(finding, vulnerability_ids_to_process, delete_existing=True) + return finding + def finding_post_processing( self, finding: Finding, finding_from_report: Finding, - ) -> None: + ) -> Finding: """ Save all associated objects to the finding after it has been saved for the purpose of foreign key restrictions @@ -715,10 +924,19 @@ def finding_post_processing( finding.unsaved_files = finding_from_report.unsaved_files self.process_files(finding) # Process vulnerability IDs - if finding_from_report.unsaved_vulnerability_ids: - finding.unsaved_vulnerability_ids = finding_from_report.unsaved_vulnerability_ids + # Copy unsaved_vulnerability_ids from the report finding to the existing finding + # so reconcile_vulnerability_ids can process them + # Always set it (even if empty list) so we can clear existing IDs when report has none + finding.unsaved_vulnerability_ids = finding_from_report.unsaved_vulnerability_ids or [] + # Store the current cve value to check if it changes + old_cve = finding.cve # legacy cve field has already been processed/set earlier - return self.process_vulnerability_ids(finding) + finding = self.reconcile_vulnerability_ids(finding) + # Save the finding only if the cve field was changed by save_vulnerability_ids + # This is temporary as the cve field will be phased out + if finding.cve != old_cve: + finding.save() + return finding def process_groups_for_all_findings( self, diff --git a/dojo/importers/endpoint_manager.py b/dojo/importers/endpoint_manager.py index ccfff345c40..f4b277d49fa 100644 --- a/dojo/importers/endpoint_manager.py +++ b/dojo/importers/endpoint_manager.py @@ -68,6 +68,7 @@ def mitigate_endpoint_status( ) -> None: """Mitigates all endpoint status objects that are supplied""" now = timezone.now() + to_update = [] for endpoint_status in endpoint_status_list: # Only mitigate endpoints that are actually active if endpoint_status.mitigated is False: @@ -75,7 +76,14 @@ def mitigate_endpoint_status( endpoint_status.last_modified = now endpoint_status.mitigated_by = user endpoint_status.mitigated = True - endpoint_status.save() + to_update.append(endpoint_status) + + if to_update: + Endpoint_Status.objects.bulk_update( + to_update, + ["mitigated", "mitigated_time", "last_modified", "mitigated_by"], + batch_size=1000, + ) @dojo_async_task @app.task() @@ -85,6 +93,8 @@ def reactivate_endpoint_status( **kwargs: dict, ) -> None: """Reactivate all endpoint status objects that are supplied""" + now = timezone.now() + to_update = [] for endpoint_status in endpoint_status_list: # Only reactivate endpoints that are actually mitigated if endpoint_status.mitigated: @@ -92,8 +102,15 @@ def reactivate_endpoint_status( endpoint_status.mitigated_by = None endpoint_status.mitigated_time = None endpoint_status.mitigated = False - endpoint_status.last_modified = timezone.now() - endpoint_status.save() + endpoint_status.last_modified = now + to_update.append(endpoint_status) + + if to_update: + Endpoint_Status.objects.bulk_update( + to_update, + ["mitigated", "mitigated_time", "mitigated_by", "last_modified"], + batch_size=1000, + ) def chunk_endpoints_and_disperse( self, @@ -149,17 +166,17 @@ def update_endpoint_status( # New finding is mitigated, so mitigate all old endpoints endpoint_status_to_mitigate = existing_finding_endpoint_status_list else: + # Convert to set for O(1) lookups instead of O(n) linear search + new_finding_endpoints_set = set(new_finding_endpoints_list) # Mitigate any endpoints in the old finding not found in the new finding - endpoint_status_to_mitigate = list( - filter( - lambda existing_finding_endpoint_status: existing_finding_endpoint_status.endpoint not in new_finding_endpoints_list, - existing_finding_endpoint_status_list), - ) + endpoint_status_to_mitigate = [ + eps for eps in existing_finding_endpoint_status_list + if eps.endpoint not in new_finding_endpoints_set + ] # Re-activate any endpoints in the old finding that are in the new finding - endpoint_status_to_reactivate = list( - filter( - lambda existing_finding_endpoint_status: existing_finding_endpoint_status.endpoint in new_finding_endpoints_list, - existing_finding_endpoint_status_list), - ) + endpoint_status_to_reactivate = [ + eps for eps in existing_finding_endpoint_status_list + if eps.endpoint in new_finding_endpoints_set + ] self.chunk_endpoints_and_reactivate(endpoint_status_to_reactivate) self.chunk_endpoints_and_mitigate(endpoint_status_to_mitigate, user) diff --git a/dojo/jira_link/views.py b/dojo/jira_link/views.py index d30681bef27..ebcc9616d4a 100644 --- a/dojo/jira_link/views.py +++ b/dojo/jira_link/views.py @@ -5,6 +5,7 @@ import re # Third party imports +import pghistory from django.contrib import messages from django.contrib.admin.utils import NestedObjects from django.core.exceptions import PermissionDenied @@ -85,85 +86,98 @@ def webhook(request, secret=None): if request.content_type != "application/json": return webhook_responser_handler("debug", "only application/json supported") # Time to process the request + # Parse the JSON first to get webhook event type for context try: parsed = json.loads(request.body.decode("utf-8")) - # Check if the events supplied are supported - if parsed.get("webhookEvent") not in {"comment_created", "jira:issue_updated"}: - return webhook_responser_handler("info", f"Unrecognized JIRA webhook event received: {parsed.get('webhookEvent')}") - - if parsed.get("webhookEvent") == "jira:issue_updated": - # xml examples at the end of file - jid = parsed["issue"]["id"] - # This may raise a 404, but it will be handled in the exception response - try: - jissue = JIRA_Issue.objects.get(jira_id=jid) - except JIRA_Instance.DoesNotExist: - return webhook_responser_handler("info", f"JIRA issue {jid} is not linked to a DefectDojo Finding") - findings = None - # Determine what type of object we will be working with - if jissue.finding: - logger.debug(f"Received issue update for {jissue.jira_key} for finding {jissue.finding.id}") - findings = [jissue.finding] - elif jissue.finding_group: - logger.debug(f"Received issue update for {jissue.jira_key} for finding group {jissue.finding_group}") - findings = jissue.finding_group.findings.all() - elif jissue.engagement: - return webhook_responser_handler("debug", "Update for engagement ignored") - else: - return webhook_responser_handler("info", f"Received issue update for {jissue.jira_key} for unknown object") - # Process the assignee if present - assignee = parsed["issue"]["fields"].get("assignee") - assignee_name = "Jira User" - if assignee is not None: - # First look for the 'name' field. If not present, try 'displayName'. Else put None - assignee_name = assignee.get("name", assignee.get("displayName")) - - # "resolution":{ - # "self":"http://www.testjira.com/rest/api/2/resolution/11", - # "id":"11", - # "description":"Cancelled by the customer.", - # "name":"Cancelled" - # }, - - # or - # "resolution": null - - # or - # "resolution": "None" - - resolution = parsed["issue"]["fields"]["resolution"] - resolution = resolution if resolution and resolution != "None" else None - resolution_id = resolution["id"] if resolution else None - resolution_name = resolution["name"] if resolution else None - jira_now = parse_datetime(parsed["issue"]["fields"]["updated"]) - - if findings: - for finding in findings: - jira_helper.process_resolution_from_jira(finding, resolution_id, resolution_name, assignee_name, jira_now, jissue, finding_group=jissue.finding_group) - # Check for any comment that could have come along with the resolution - if (error_response := check_for_and_create_comment(parsed)) is not None: - return error_response - - if parsed.get("webhookEvent") == "comment_created": - if (error_response := check_for_and_create_comment(parsed)) is not None: - return error_response - except Exception as e: - # Check if the issue is originally a 404 - if isinstance(e, Http404): - return webhook_responser_handler("debug", str(e)) - # Try to get a little more information on the exact exception + return webhook_responser_handler("debug", f"Failed to parse JSON: {e}") + + # Check if the events supplied are supported + if parsed.get("webhookEvent") not in {"comment_created", "jira:issue_updated"}: + return webhook_responser_handler("info", f"Unrecognized JIRA webhook event received: {parsed.get('webhookEvent')}") + + # Wrap processing with pghistory context for audit trail + # JIRA webhooks don't have a user session, so we create a new context + with pghistory.context( + source="jira_webhook", + jira_event=parsed.get("webhookEvent"), + ): try: - message = ( - f"Original Exception: {e}\n" - f"jira webhook body parsed:\n{json.dumps(parsed, indent=4)}" - ) - except Exception: - message = ( - f"Original Exception: {e}\n" - f"jira webhook body :\n{request.body.decode('utf-8')}" - ) - return webhook_responser_handler("debug", message) + if parsed.get("webhookEvent") == "jira:issue_updated": + # xml examples at the end of file + jid = parsed["issue"]["id"] + # This may raise a 404, but it will be handled in the exception response + try: + jissue = JIRA_Issue.objects.get(jira_id=jid) + except JIRA_Instance.DoesNotExist: + return webhook_responser_handler("info", f"JIRA issue {jid} is not linked to a DefectDojo Finding") + # Add jira_key to context now that we have it + pghistory.context(jira_key=jissue.jira_key) + findings = None + # Determine what type of object we will be working with + if jissue.finding: + logger.debug(f"Received issue update for {jissue.jira_key} for finding {jissue.finding.id}") + findings = [jissue.finding] + elif jissue.finding_group: + logger.debug(f"Received issue update for {jissue.jira_key} for finding group {jissue.finding_group}") + findings = jissue.finding_group.findings.all() + elif jissue.engagement: + return webhook_responser_handler("debug", "Update for engagement ignored") + else: + return webhook_responser_handler("info", f"Received issue update for {jissue.jira_key} for unknown object") + # Process the assignee if present + assignee = parsed["issue"]["fields"].get("assignee") + assignee_name = "Jira User" + if assignee is not None: + # First look for the 'name' field. If not present, try 'displayName'. Else put None + assignee_name = assignee.get("name", assignee.get("displayName")) + + # "resolution":{ + # "self":"http://www.testjira.com/rest/api/2/resolution/11", + # "id":"11", + # "description":"Cancelled by the customer.", + # "name":"Cancelled" + # }, + + # or + # "resolution": null + + # or + # "resolution": "None" + + resolution = parsed["issue"]["fields"]["resolution"] + resolution = resolution if resolution and resolution != "None" else None + resolution_id = resolution["id"] if resolution else None + resolution_name = resolution["name"] if resolution else None + jira_now = parse_datetime(parsed["issue"]["fields"]["updated"]) + + if findings: + for finding in findings: + jira_helper.process_resolution_from_jira(finding, resolution_id, resolution_name, assignee_name, jira_now, jissue, finding_group=jissue.finding_group) + # Check for any comment that could have come along with the resolution + if (error_response := check_for_and_create_comment(parsed)) is not None: + return error_response + + if parsed.get("webhookEvent") == "comment_created": + if (error_response := check_for_and_create_comment(parsed)) is not None: + return error_response + + except Exception as e: + # Check if the issue is originally a 404 + if isinstance(e, Http404): + return webhook_responser_handler("debug", str(e)) + # Try to get a little more information on the exact exception + try: + message = ( + f"Original Exception: {e}\n" + f"jira webhook body parsed:\n{json.dumps(parsed, indent=4)}" + ) + except Exception: + message = ( + f"Original Exception: {e}\n" + f"jira webhook body :\n{request.body.decode('utf-8')}" + ) + return webhook_responser_handler("debug", message) return webhook_responser_handler("No logging here", "Success!") diff --git a/dojo/labels.py b/dojo/labels.py index ce5ea520c5d..504c8e55fe1 100644 --- a/dojo/labels.py +++ b/dojo/labels.py @@ -63,11 +63,11 @@ def __init__(self, label_set: dict[str, str]): As a side benefit, this will explode if any label defined on this class is not present in the given dict: a runtime check that a labels dict must be complete. """ - for _l, _v in self._get_label_entries().items(): + for l_, v_ in self._get_label_entries().items(): try: - setattr(self, _l, label_set[_v]) + setattr(self, l_, label_set[v_]) except KeyError: - error_message = f"Supplied copy dictionary does not provide entry for {_l}" + error_message = f"Supplied copy dictionary does not provide entry for {l_}" logger.error(error_message) raise ValueError(error_message) diff --git a/dojo/management/commands/complete_initialization.py b/dojo/management/commands/complete_initialization.py new file mode 100644 index 00000000000..556c77867fb --- /dev/null +++ b/dojo/management/commands/complete_initialization.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import os +import secrets +import string +import uuid +from pathlib import Path +from typing import Any + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.management import BaseCommand, CommandError, call_command +from django.db import connection, connections +from django.db.utils import ProgrammingError + +from dojo.auditlog import configure_pghistory_triggers +from dojo.models import Announcement, Dojo_User, UserAnnouncement + + +class Command(BaseCommand): + help = "Initialize DefectDojo application state" + + def handle(self, *args: Any, **options: Any) -> None: + if os.getenv("DD_INITIALIZE") == "false": + self.stdout.write("Initialization skipped (DD_INITIALIZE=false)") + return + + self.stdout.write("Initializing DefectDojo") + + self.check_enable_auditlog_consistency() + self.warn_on_missing_migrations() + + self.stdout.write("Applying migrations") + call_command("migrate", interactive=False) + + self.stdout.write("Configuring pghistory triggers") + configure_pghistory_triggers() + + if self.admin_user_exists(): + self.stdout.write("Admin user already exists; skipping first-boot setup") + self.create_announcement_banner() + self.initialize_data() + return + + self.ensure_admin_secrets() + self.first_boot_setup() + self.create_announcement_banner() + self.initialize_data() + + # ------------------------------------------------------------------ + # Initialization steps + # ------------------------------------------------------------------ + + def initialize_data(self) -> None: + self.stdout.write("Initializing test types") + call_command("initialize_test_types") + + self.stdout.write("Initializing non-standard permissions") + call_command("initialize_permissions") + + def create_announcement_banner(self) -> None: + if os.getenv("DD_CREATE_CLOUD_BANNER"): + return + + self.stdout.write("Creating announcement banner") + + announcement, _ = Announcement.objects.get_or_create(id=1) + announcement.message = ( + '' + "DefectDojo Pro Cloud and On-Premise Subscriptions Now Available! " + "Create an account to try Pro for free!" + "" + ) + announcement.dismissable = True + announcement.save() + + for user in Dojo_User.objects.all(): + UserAnnouncement.objects.get_or_create( + user=user, + announcement=announcement, + ) + + # ------------------------------------------------------------------ + # Auditlog consistency + # ------------------------------------------------------------------ + + def check_enable_auditlog_consistency(self) -> None: + self.stdout.write("Checking ENABLE_AUDITLOG consistency") + + try: + with connections["default"].cursor() as cursor: + try: + cursor.execute("SELECT * FROM dojo_system_settings LIMIT 1") + except ProgrammingError as exc: + msg = str(exc) + if "does not exist" in msg or "doesn't exist" in msg: + self.stdout.write("Database not initialized yet; skipping auditlog check") + return + raise + row = dict(zip([col[0] for col in cursor.description], cursor.fetchone(), strict=False)) + + except Exception as exc: + msg = f"Failed to read system settings from database: {exc}" + raise CommandError(msg) from exc + + if not row.get("enable_auditlog", True) and settings.ENABLE_AUDITLOG: + msg = "Auditlog disabled in DB but ENABLE_AUDITLOG=True. Set DD_ENABLE_AUDITLOG=False for all Django containers." + raise CommandError(msg) + + # ------------------------------------------------------------------ + # Migration checks (warning only) + # ------------------------------------------------------------------ + + def warn_on_missing_migrations(self) -> None: + self.stdout.write("Checking for missing migrations") + + try: + call_command( + "makemigrations", + "--check", + "--dry-run", + verbosity=3, + ) + except SystemExit: + self.stderr.write( + "\n" + "********************************************************************************\n" + "WARNING: Missing Database Migrations Detected\n" + "********************************************************************************\n" + "You made changes to models without creating migrations.\n\n" + "Startup will continue, but you should fix this.\n" + "********************************************************************************\n", + ) + + # ------------------------------------------------------------------ + # Admin / first boot + # ------------------------------------------------------------------ + + def admin_user_exists(self) -> bool: + username = os.getenv("DD_ADMIN_USER") + if not username: + msg = "DD_ADMIN_USER is not set" + raise CommandError(msg) + + User = get_user_model() + return User.objects.filter(username=username).exists() + + def ensure_admin_secrets(self) -> None: + if not os.getenv("DD_ADMIN_PASSWORD"): + password = self.generate_password(22) + os.environ["DD_ADMIN_PASSWORD"] = password + self.stdout.write(f"Admin password: {password}") + + if not os.getenv("DD_JIRA_WEBHOOK_SECRET"): + secret = str(uuid.uuid4()) + os.environ["DD_JIRA_WEBHOOK_SECRET"] = secret + self.stdout.write(f"JIRA Webhook Secret: {secret}") + + def first_boot_setup(self) -> None: + self.stdout.write("Running first boot setup") + + self.create_admin_user() + self.load_initial_fixtures() + self.persist_jira_webhook_secret() + self.load_extra_fixtures() + self.install_watson() + call_command("migrate_textquestions") + + def create_admin_user(self) -> None: + + User = get_user_model() + username = os.getenv("DD_ADMIN_USER") + + if User.objects.filter(username=username).exists(): + self.stdout.write(f"Admin user '{username}' already exists; skipping creation") + return + + User.objects.create_superuser( + username, + os.getenv("DD_ADMIN_MAIL"), + os.getenv("DD_ADMIN_PASSWORD"), + first_name=os.getenv("DD_ADMIN_FIRST_NAME"), + last_name=os.getenv("DD_ADMIN_LAST_NAME"), + ) + + # ------------------------------------------------------------------ + # Fixtures & setup + # ------------------------------------------------------------------ + + def load_initial_fixtures(self) -> None: + self.stdout.write("Loading initial fixtures") + call_command( + "loaddata", + "system_settings", + "initial_banner_conf", + "product_type", + "test_type", + "development_environment", + "benchmark_type", + "benchmark_category", + "benchmark_requirement", + "language_type", + "objects_review", + "regulation", + "initial_surveys", + "role", + "sla_configurations", + ) + + def persist_jira_webhook_secret(self) -> None: + with connection.cursor() as cursor: + cursor.execute( + "UPDATE dojo_system_settings SET jira_webhook_secret = %s", + [os.getenv("DD_JIRA_WEBHOOK_SECRET")], + ) + + def load_extra_fixtures(self) -> None: + for fixture in sorted(Path("dojo/fixtures").glob("extra_*.json")): + self.stdout.write(f"Loading {fixture}") + call_command("loaddata", fixture.stem) + + def install_watson(self) -> None: + self.stdout.write("Installing watson search index") + call_command("installwatson") + + # ------------------------------------------------------------------ + # Utilities + # ------------------------------------------------------------------ + + def generate_password(self, length: int) -> str: + alphabet = string.ascii_letters + string.digits + return "".join(secrets.choice(alphabet) for _ in range(length)) diff --git a/dojo/management/commands/dedupe.py b/dojo/management/commands/dedupe.py index a8e0a538cfe..3eddccdc45d 100644 --- a/dojo/management/commands/dedupe.py +++ b/dojo/management/commands/dedupe.py @@ -1,5 +1,6 @@ import logging +import pghistory from django.conf import settings from django.core.management.base import BaseCommand from django.db.models import Prefetch @@ -64,6 +65,21 @@ def handle(self, *args, **options): dedupe_sync = options["dedupe_sync"] dedupe_batch_mode = options.get("dedupe_batch_mode", True) # Default to True (batch mode enabled) + # Wrap with pghistory context for audit trail + with pghistory.context( + source="dedupe_command", + dedupe_sync=dedupe_sync, + ): + self._run_dedupe( + restrict_to_parsers=restrict_to_parsers, + hash_code_only=hash_code_only, + dedupe_only=dedupe_only, + dedupe_sync=dedupe_sync, + dedupe_batch_mode=dedupe_batch_mode, + ) + + def _run_dedupe(self, *, restrict_to_parsers, hash_code_only, dedupe_only, dedupe_sync, dedupe_batch_mode): + """Internal method to run the dedupe logic within pghistory context.""" if restrict_to_parsers is not None: findings = Finding.objects.filter(test__test_type__name__in=restrict_to_parsers).exclude(duplicate=True) logger.info("######## Will process only parsers %s and %d findings ########", *restrict_to_parsers, findings.count()) diff --git a/dojo/management/commands/force_token_reset.py b/dojo/management/commands/force_token_reset.py new file mode 100644 index 00000000000..fc00dc8c2e3 --- /dev/null +++ b/dojo/management/commands/force_token_reset.py @@ -0,0 +1,46 @@ +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand, CommandError + +from dojo.user.authentication import reset_token_for_user + + +class Command(BaseCommand): + help = "Rotate (reset) the DRF API token for a target user. Requires an acting user (superuser or global owner)." + + def add_arguments(self, parser): + parser.add_argument("--acting-user", required=True, help="Username of the acting user performing the reset.") + parser.add_argument("--username", help="Username of the target user.") + parser.add_argument("--user-id", type=int, help="ID of the target user.") + + def handle(self, *args, **options): + User = get_user_model() + + acting_username = options["acting_user"] + target_username = options.get("username") + target_user_id = options.get("user_id") + + if bool(target_username) == bool(target_user_id): + msg = "Provide exactly one of --username or --user-id." + raise CommandError(msg) + + try: + acting_user = User.objects.get(username=acting_username) + except User.DoesNotExist as exc: + msg = f"Acting user '{acting_username}' does not exist." + raise CommandError(msg) from exc + + try: + if target_username: + target_user = User.objects.get(username=target_username) + else: + target_user = User.objects.get(id=target_user_id) + except User.DoesNotExist as exc: + msg = "Target user does not exist." + raise CommandError(msg) from exc + + try: + reset_token_for_user(acting_user=acting_user, target_user=target_user) + except Exception as exc: + raise CommandError(str(exc)) from exc + + self.stdout.write(self.style.SUCCESS("API token reset successfully.")) diff --git a/dojo/management/commands/jira_status_reconciliation.py b/dojo/management/commands/jira_status_reconciliation.py index c8ce694015d..a6a49e9256b 100644 --- a/dojo/management/commands/jira_status_reconciliation.py +++ b/dojo/management/commands/jira_status_reconciliation.py @@ -1,5 +1,6 @@ import logging +import pghistory from dateutil.relativedelta import relativedelta from django.conf import settings from django.core.management.base import BaseCommand @@ -216,10 +217,9 @@ def add_arguments(self, parser): parser.add_argument("--dryrun", action="store_true", help="Only print actions to be performed, but make no modifications.") def handle(self, *args, **options): - # mode = options['mode'] - # product = options['product'] - # engagement = options['engagement'] - # daysback = options['daysback'] - # dryrun = options['dryrun'] - - return jira_status_reconciliation(*args, **options) + # Wrap with pghistory context for audit trail + with pghistory.context( + source="jira_reconciliation", + mode=options.get("mode", "reconcile"), + ): + return jira_status_reconciliation(*args, **options) diff --git a/dojo/management/commands/list_top_tests.py b/dojo/management/commands/list_top_tests.py new file mode 100644 index 00000000000..7666d707101 --- /dev/null +++ b/dojo/management/commands/list_top_tests.py @@ -0,0 +1,119 @@ +from django.core.management.base import BaseCommand +from django.db.models import Count, Q + +from dojo.models import Test + + +class Command(BaseCommand): + help = "List the top 25 tests with the most findings" + + def add_arguments(self, parser): + parser.add_argument( + "--limit", + type=int, + default=25, + help="Number of tests to display (default: 25)", + ) + + def handle(self, *args, **options): + limit = options["limit"] + + # Annotate tests with finding counts + tests = ( + Test.objects.annotate( + total_findings=Count("finding", distinct=True), + active_findings=Count("finding", filter=Q(finding__active=True), distinct=True), + duplicate_findings=Count("finding", filter=Q(finding__duplicate=True), distinct=True), + ) + .filter(total_findings__gt=0) + .select_related("engagement", "engagement__product", "test_type") + .order_by("-total_findings")[:limit] + ) + + if not tests: + self.stdout.write(self.style.WARNING("No tests with findings found.")) + return + + # Calculate column widths + max_test_id_len = max( + (len(str(test.id)) for test in tests), + default=8, + ) + max_product_len = max( + (len(str(test.engagement.product.name)) for test in tests), + default=20, + ) + max_engagement_len = max( + (len(str(test.engagement.name)) for test in tests), + default=20, + ) + max_test_len = max( + (len(str(test.title or test.id)) for test in tests), + default=20, + ) + max_test_type_len = max( + (len(str(test.test_type.name)) for test in tests), + default=20, + ) + max_dedup_algo_len = max( + (len(str(test.deduplication_algorithm)) for test in tests), + default=20, + ) + + # Ensure minimum widths for readability + max_test_id_len = max(max_test_id_len, 8) + max_product_len = max(max_product_len, 20) + max_engagement_len = max(max_engagement_len, 20) + max_test_len = max(max_test_len, 20) + max_test_type_len = max(max_test_type_len, 20) + max_dedup_algo_len = max(max_dedup_algo_len, 20) + + # Header + header = ( + f"{'Test ID':<{max_test_id_len}} | " + f"{'Product':<{max_product_len}} | " + f"{'Engagement':<{max_engagement_len}} | " + f"{'Test':<{max_test_len}} | " + f"{'Test Type':<{max_test_type_len}} | " + f"{'Dedup Algorithm':<{max_dedup_algo_len}} | " + f"{'Total':>8} | " + f"{'Active':>8} | " + f"{'Duplicate':>10}" + ) + separator = "-" * len(header) + + self.stdout.write(self.style.SUCCESS(header)) + self.stdout.write(separator) + + # Data rows + for test in tests: + test_id = str(test.id) + product_name = str(test.engagement.product.name) + engagement_name = str(test.engagement.name) + test_name = str(test.title or f"Test #{test.id}") + test_type_name = str(test.test_type.name) + dedup_algo = str(test.deduplication_algorithm) + total = test.total_findings + active = test.active_findings + duplicate = test.duplicate_findings + + row = ( + f"{test_id:<{max_test_id_len}} | " + f"{product_name:<{max_product_len}} | " + f"{engagement_name:<{max_engagement_len}} | " + f"{test_name:<{max_test_len}} | " + f"{test_type_name:<{max_test_type_len}} | " + f"{dedup_algo:<{max_dedup_algo_len}} | " + f"{total:>8} | " + f"{active:>8} | " + f"{duplicate:>10}" + ) + self.stdout.write(row) + + # Summary + self.stdout.write(separator) + self.stdout.write( + self.style.SUCCESS( + f"\nDisplayed top {len(tests)} tests with findings.", + ), + ) diff --git a/dojo/management/commands/migrate_cve.py b/dojo/management/commands/migrate_cve.py index c5d0a1fdc62..eaa3985c19c 100644 --- a/dojo/management/commands/migrate_cve.py +++ b/dojo/management/commands/migrate_cve.py @@ -2,11 +2,11 @@ from django.core.management.base import BaseCommand +from dojo.finding.helper import save_vulnerability_ids_template from dojo.models import ( Finding, Finding_Template, Vulnerability_Id, - Vulnerability_Id_Template, ) from dojo.utils import mass_model_updater @@ -20,9 +20,9 @@ def create_vulnerability_id(finding): def create_vulnerability_id_template(finding_template): - Vulnerability_Id_Template.objects.get_or_create( - finding_template=finding_template, vulnerability_id=finding_template.cve, - ) + # Use the new TextField-based approach + if finding_template.cve: + save_vulnerability_ids_template(finding_template, [finding_template.cve]) class Command(BaseCommand): diff --git a/dojo/management/commands/pghistory_backfill.py b/dojo/management/commands/pghistory_backfill.py index 456cbe75c5d..751799831a2 100644 --- a/dojo/management/commands/pghistory_backfill.py +++ b/dojo/management/commands/pghistory_backfill.py @@ -37,12 +37,7 @@ def add_arguments(self, parser): parser.add_argument( "--log-queries", action="store_true", - help="Enable database query logging (default: enabled)", - ) - parser.add_argument( - "--no-log-queries", - action="store_true", - help="Disable database query logging", + help="Enable database query logging (default: disabled)", ) def get_excluded_fields(self, model_name): @@ -140,31 +135,27 @@ def disable_db_logging(self): ) def handle(self, *args, **options): - if not settings.ENABLE_AUDITLOG or settings.AUDITLOG_TYPE != "django-pghistory": + if not settings.ENABLE_AUDITLOG: self.stdout.write( self.style.WARNING( - "pghistory is not enabled. Set DD_ENABLE_AUDITLOG=True and " - "DD_AUDITLOG_TYPE=django-pghistory", + "pghistory is not enabled. Set DD_ENABLE_AUDITLOG=True", ), ) return # Enable database query logging based on options - # Default to enabled unless explicitly disabled - enable_query_logging = not options.get("no_log_queries") + # Default to disabled unless explicitly enabled + enable_query_logging = options.get("log_queries", False) if enable_query_logging: self.enable_db_logging() - else: - self.stdout.write( - self.style.WARNING("Database query logging disabled"), - ) # Models that are tracked by pghistory tracked_models = [ "Dojo_User", "Endpoint", "Engagement", "Finding", "Finding_Group", "Product_Type", "Product", "Test", "Risk_Acceptance", "Finding_Template", "Cred_User", "Notification_Webhooks", + "FindingReviewers", # M2M through table for Finding.reviewers ] specific_model = options.get("model") @@ -220,23 +211,23 @@ def handle(self, *args, **options): ) continue - # Get IDs of records that already have initial_import events - existing_initial_import_ids = set( - EventModel.objects.filter(pgh_label="initial_import").values_list("pgh_obj_id", flat=True), + # Get IDs of records that already have initial_backfill events + existing_initial_backfill_ids = set( + EventModel.objects.filter(pgh_label="initial_backfill").values_list("pgh_obj_id", flat=True), ) - # Filter to only get records that don't have initial_import events - records_needing_backfill = Model.objects.exclude(id__in=existing_initial_import_ids) + # Filter to only get records that don't have initial_backfill events + records_needing_backfill = Model.objects.exclude(id__in=existing_initial_backfill_ids) backfill_count = records_needing_backfill.count() - existing_count = len(existing_initial_import_ids) + existing_count = len(existing_initial_backfill_ids) # Log the breakdown - self.stdout.write(f" Records with initial_import events: {existing_count:,}") - self.stdout.write(f" Records needing initial_import events: {backfill_count:,}") + self.stdout.write(f" Records with initial_backfill events: {existing_count:,}") + self.stdout.write(f" Records needing initial_backfill events: {backfill_count:,}") if backfill_count == 0: self.stdout.write( - self.style.SUCCESS(f" ✓ All {total_count:,} records already have initial_import events"), + self.style.SUCCESS(f" ✓ All {total_count:,} records already have initial_backfill events"), ) processed = total_count continue @@ -284,7 +275,7 @@ def handle(self, *args, **options): # Add pghistory-specific fields event_data.update({ - "pgh_label": "initial_import", + "pgh_label": "initial_backfill", "pgh_obj": instance, # ForeignKey to the original object "pgh_context": None, # No context for backfilled events }) diff --git a/dojo/management/commands/pghistory_backfill_fast.py b/dojo/management/commands/pghistory_backfill_fast.py index a2f1921fc74..23682ad6c62 100644 --- a/dojo/management/commands/pghistory_backfill_fast.py +++ b/dojo/management/commands/pghistory_backfill_fast.py @@ -4,14 +4,16 @@ This command creates initial snapshots for all existing records in tracked models using PostgreSQL COPY for maximum performance. """ -import io import logging import time from django.conf import settings from django.core.management.base import BaseCommand -from django.db import connection -from django.utils import timezone + +from dojo.auditlog import ( + get_tracked_models, + process_model_backfill, +) logger = logging.getLogger(__name__) @@ -39,377 +41,28 @@ def add_arguments(self, parser): parser.add_argument( "--log-queries", action="store_true", - help="Enable database query logging (default: enabled)", + help="Enable database query logging (default: disabled)", ) - parser.add_argument( - "--no-log-queries", - action="store_true", - help="Disable database query logging", - ) - - def get_excluded_fields(self, model_name): - """Get the list of excluded fields for a specific model from pghistory configuration.""" - # Define excluded fields for each model (matching auditlog.py) - excluded_fields_map = { - "Dojo_User": ["password"], - "Product": ["updated"], # This is the key change - "Cred_User": ["password"], - "Notification_Webhooks": ["header_name", "header_value"], - } - return excluded_fields_map.get(model_name, []) def process_model_with_copy(self, model_name, batch_size, dry_run): """Process a single model using COPY operations with raw SQL.""" - try: - # Get table names using raw SQL - # Handle special cases for table naming - if model_name == "Dojo_User": - table_name = "dojo_dojo_user" - event_table_name = "dojo_dojo_userevent" - elif model_name == "Product_Type": - table_name = "dojo_product_type" - event_table_name = "dojo_product_typeevent" - elif model_name == "Finding_Group": - table_name = "dojo_finding_group" - event_table_name = "dojo_finding_groupevent" - elif model_name == "Risk_Acceptance": - table_name = "dojo_risk_acceptance" - event_table_name = "dojo_risk_acceptanceevent" - elif model_name == "Finding_Template": - table_name = "dojo_finding_template" - event_table_name = "dojo_finding_templateevent" - elif model_name == "Cred_User": - table_name = "dojo_cred_user" - event_table_name = "dojo_cred_userevent" - elif model_name == "Notification_Webhooks": - table_name = "dojo_notification_webhooks" - event_table_name = "dojo_notification_webhooksevent" + def progress_callback(msg, style=None): + """Progress callback that uses self.stdout.write with styling.""" + if style == "SUCCESS": + self.stdout.write(self.style.SUCCESS(msg)) + elif style == "ERROR": + self.stdout.write(self.style.ERROR(msg)) + elif style == "WARNING": + self.stdout.write(self.style.WARNING(msg)) else: - table_name = f"dojo_{model_name.lower()}" - event_table_name = f"dojo_{model_name.lower()}event" - - # Check if tables exist - with connection.cursor() as cursor: - cursor.execute(""" - SELECT EXISTS ( - SELECT FROM information_schema.tables - WHERE table_name = %s - ) - """, [table_name]) - table_exists = cursor.fetchone()[0] - - cursor.execute(""" - SELECT EXISTS ( - SELECT FROM information_schema.tables - WHERE table_name = %s - ) - """, [event_table_name]) - event_table_exists = cursor.fetchone()[0] - - if not table_exists: - self.stdout.write(f" Table {table_name} not found") - return 0, 0.0 - - if not event_table_exists: - self.stdout.write( - self.style.ERROR( - f" Event table {event_table_name} not found. " - f"Is {model_name} tracked by pghistory?", - ), - ) - return 0, 0.0 - - # Get total count using raw SQL - with connection.cursor() as cursor: - cursor.execute(f"SELECT COUNT(*) FROM {table_name}") - total_count = cursor.fetchone()[0] - - if total_count == 0: - self.stdout.write(f" No records found for {model_name}") - return 0, 0.0 - - self.stdout.write(f" Found {total_count:,} records") - - # Get excluded fields - excluded_fields = self.get_excluded_fields(model_name) - - # Check if records already have initial_import events using raw SQL - with connection.cursor() as cursor: - cursor.execute(f"SELECT COUNT(*) FROM {event_table_name} WHERE pgh_label = 'initial_import'") - existing_count = cursor.fetchone()[0] - - # Get records that need backfill using raw SQL - with connection.cursor() as cursor: - cursor.execute(f""" - SELECT COUNT(*) FROM {table_name} t - WHERE NOT EXISTS ( - SELECT 1 FROM {event_table_name} e - WHERE e.pgh_obj_id = t.id AND e.pgh_label = 'initial_import' - ) - """) - backfill_count = cursor.fetchone()[0] - - # Log the breakdown - self.stdout.write(f" Records with initial_import events: {existing_count:,}") - self.stdout.write(f" Records needing initial_import events: {backfill_count:,}") - - if backfill_count == 0: - self.stdout.write( - self.style.SUCCESS(f" ✓ All {total_count:,} records already have initial_import events"), - ) - return total_count, 0.0 - - if dry_run: - self.stdout.write(f" Would process {backfill_count:,} records using COPY...") - return backfill_count, 0.0 - - # Get event table columns using raw SQL (excluding auto-generated pgh_id) - with connection.cursor() as cursor: - cursor.execute(""" - SELECT column_name - FROM information_schema.columns - WHERE table_name = %s AND column_name != 'pgh_id' - ORDER BY ordinal_position - """, [event_table_name]) - event_columns = [row[0] for row in cursor.fetchall()] - - # Get all IDs that need backfill first - with connection.cursor() as cursor: - cursor.execute(f""" - SELECT t.id FROM {table_name} t - WHERE NOT EXISTS ( - SELECT 1 FROM {event_table_name} e - WHERE e.pgh_obj_id = t.id AND e.pgh_label = 'initial_import' - ) - ORDER BY t.id - """) - ids_to_process = [row[0] for row in cursor.fetchall()] - - if not ids_to_process: - self.stdout.write(" No records need backfill") - return 0, 0.0 - - # Process records in batches using raw SQL - processed = 0 - batch_start_time = time.time() - model_start_time = time.time() # Track model start time - - # Get column names for the source table - with connection.cursor() as cursor: - cursor.execute(""" - SELECT column_name - FROM information_schema.columns - WHERE table_name = %s - ORDER BY ordinal_position - """, [table_name]) - source_columns = [row[0] for row in cursor.fetchall()] - - # Filter out excluded fields from source columns - source_columns = [col for col in source_columns if col not in excluded_fields] - - # Process in batches - consecutive_failures = 0 - max_failures = 3 - - for i in range(0, len(ids_to_process), batch_size): - batch_ids = ids_to_process[i:i + batch_size] - - # Log progress every 10 batches - if i > 0 and i % (batch_size * 10) == 0: - self.stdout.write(f" Processing batch starting at index {i:,}...") - - # Get batch of records using raw SQL with specific IDs - columns_str = ", ".join(source_columns) - placeholders = ", ".join(["%s"] * len(batch_ids)) - query = f""" - SELECT {columns_str} FROM {table_name} t - WHERE t.id IN ({placeholders}) - ORDER BY t.id - """ - - with connection.cursor() as cursor: - cursor.execute(query, batch_ids) - batch_rows = cursor.fetchall() - - if not batch_rows: - self.stdout.write(f" No records found for batch at index {i}") - continue - - # Use PostgreSQL COPY as described in the article - try: - # Prepare data for COPY using a custom file-like object - class FileLikeObject: - def __init__(self): - self.data = io.BytesIO() - - def write(self, data): - return self.data.write(data) - - def read(self, size=-1): - return self.data.read(size) - - def seek(self, pos): - return self.data.seek(pos) - - def tell(self): - return self.data.tell() - - def __len__(self): - return len(self.data.getvalue()) - - def getvalue(self): - return self.data.getvalue() - - copy_buffer = FileLikeObject() - - for row in batch_rows: - row_data = [] - - # Create a mapping of source columns to values - source_values = {} - for idx, value in enumerate(row): - field_name = source_columns[idx] - # Convert value to string for COPY - if value is None: - source_values[field_name] = "" - elif isinstance(value, bool): - source_values[field_name] = "t" if value else "f" - elif hasattr(value, "isoformat"): # datetime objects - source_values[field_name] = value.isoformat() - else: - source_values[field_name] = str(value) - - # Build row data in the order of event_columns - for col in event_columns: - if col == "pgh_created_at": - row_data.append(timezone.now().isoformat()) - elif col == "pgh_label": - row_data.append("initial_import") - elif col == "pgh_obj_id": - row_data.append(str(row[0]) if row[0] is not None else "") # Assuming first column is id - elif col == "pgh_context_id": - row_data.append("") # Empty for backfilled events - elif col in source_values: - row_data.append(source_values[col]) - else: - row_data.append("") # Default empty value - - # Write tab-separated row to buffer as bytes - copy_buffer.write(("\t".join(row_data) + "\n").encode("utf-8")) - - copy_buffer.seek(0) - - # Debug: Show what we're about to copy - self.stdout.write(f" Batch {i // batch_size + 1}: Writing to table: {event_table_name}") - - # Use PostgreSQL COPY with psycopg3 syntax - with connection.cursor() as cursor: - # Get the underlying raw cursor to bypass Django's wrapper - raw_cursor = cursor.cursor - # Use the copy method (psycopg3 syntax) - copy_sql = f"COPY {event_table_name} ({', '.join(event_columns)}) FROM STDIN WITH (FORMAT text, DELIMITER E'\\t')" - - try: - # Use psycopg3 copy syntax as per documentation - # Prepare data as list of tuples for write_row() - records = [] - for row in batch_rows: - row_data = [] - - # Create a mapping of source columns to values - source_values = {} - for idx, value in enumerate(row): - field_name = source_columns[idx] - source_values[field_name] = value - - # Build row data in the order of event_columns - for col in event_columns: - if col == "pgh_created_at": - row_data.append(timezone.now()) - elif col == "pgh_label": - row_data.append("initial_import") - elif col == "pgh_obj_id": - row_data.append(row[0]) # Assuming first column is id - elif col == "pgh_context_id": - row_data.append(None) # Empty for backfilled events - elif col in source_values: - row_data.append(source_values[col]) - else: - row_data.append(None) # Default NULL value - - records.append(tuple(row_data)) - - # Use COPY with write_row() as per psycopg3 docs - with raw_cursor.copy(copy_sql) as copy: - for record in records: - copy.write_row(record) - self.stdout.write(" COPY operation completed using write_row") - - # Commit the transaction to persist the data - raw_cursor.connection.commit() - - # Debug: Check if data was inserted - raw_cursor.execute(f"SELECT COUNT(*) FROM {event_table_name} WHERE pgh_label = 'initial_import'") - count = raw_cursor.fetchone()[0] - self.stdout.write(f" Records in event table after batch: {count}") - - except Exception as copy_error: - self.stdout.write(f" COPY error: {copy_error}") - # Try to get more details about the error - raw_cursor.execute("SELECT * FROM pg_stat_activity WHERE state = 'active'") - self.stdout.write(f" Active queries: {raw_cursor.fetchall()}") - raise - - batch_processed = len(batch_rows) - processed += batch_processed - consecutive_failures = 0 # Reset failure counter on success - - # Calculate timing - batch_end_time = time.time() - batch_duration = batch_end_time - batch_start_time - batch_records_per_second = batch_processed / batch_duration if batch_duration > 0 else 0 - - # Log progress - progress = (processed / backfill_count) * 100 - self.stdout.write(f" Processed {processed:,}/{backfill_count:,} records ({progress:.1f}%) - " - f"Last batch: {batch_duration:.2f}s ({batch_records_per_second:.1f} records/sec)") - - batch_start_time = time.time() # Reset for next batch - - except Exception as e: - consecutive_failures += 1 - logger.error(f"Bulk insert failed for {model_name} batch: {e}") - self.stdout.write(f" Bulk insert failed: {e}") - # Log more details about the error - self.stdout.write(f" Processed {processed:,} records before failure") - - if consecutive_failures >= max_failures: - self.stdout.write(f" Too many consecutive failures ({consecutive_failures}), stopping processing") - break - - # Continue with next batch instead of breaking - continue - - # Calculate total timing - model_end_time = time.time() - total_duration = model_end_time - model_start_time - records_per_second = processed / total_duration if total_duration > 0 else 0 - - self.stdout.write( - self.style.SUCCESS( - f" ✓ Completed {model_name}: {processed:,} records in {total_duration:.2f}s " - f"({records_per_second:.1f} records/sec)", - ), - ) + self.stdout.write(msg) - return processed, records_per_second # noqa: TRY300 - - except Exception as e: - self.stdout.write( - self.style.ERROR(f" ✗ Failed to process {model_name}: {e}"), - ) - logger.exception(f"Error processing {model_name}") - return 0, 0.0 + return process_model_backfill( + model_name=model_name, + batch_size=batch_size, + dry_run=dry_run, + progress_callback=progress_callback, + ) def enable_db_logging(self): """Enable database query logging for this command.""" @@ -457,11 +110,10 @@ def disable_db_logging(self): ) def handle(self, *args, **options): - if not settings.ENABLE_AUDITLOG or settings.AUDITLOG_TYPE != "django-pghistory": + if not settings.ENABLE_AUDITLOG: self.stdout.write( self.style.WARNING( - "pghistory is not enabled. Set DD_ENABLE_AUDITLOG=True and " - "DD_AUDITLOG_TYPE=django-pghistory", + "pghistory is not enabled. Set DD_ENABLE_AUDITLOG=True", ), ) return @@ -477,7 +129,7 @@ def handle(self, *args, **options): return # Enable database query logging based on options - enable_query_logging = not options.get("no_log_queries") + enable_query_logging = options.get("log_queries") if enable_query_logging: self.enable_db_logging() @@ -487,11 +139,7 @@ def handle(self, *args, **options): ) # Models that are tracked by pghistory - tracked_models = [ - "Dojo_User", "Endpoint", "Engagement", "Finding", "Finding_Group", - "Product_Type", "Product", "Test", "Risk_Acceptance", - "Finding_Template", "Cred_User", "Notification_Webhooks", - ] + tracked_models = get_tracked_models() specific_model = options.get("model") if specific_model: @@ -518,7 +166,6 @@ def handle(self, *args, **options): self.stdout.write(f"Starting backfill for {len(tracked_models)} model(s) using PostgreSQL COPY...") for model_name in tracked_models: - time.time() self.stdout.write(f"\nProcessing {model_name}...") processed, _ = self.process_model_with_copy( diff --git a/dojo/management/commands/pghistory_backfill_simple.py b/dojo/management/commands/pghistory_backfill_simple.py index 0203b5506a7..ebc4d452473 100644 --- a/dojo/management/commands/pghistory_backfill_simple.py +++ b/dojo/management/commands/pghistory_backfill_simple.py @@ -119,7 +119,7 @@ def process_model_simple(self, model_name, batch_size, dry_run): SELECT COUNT(*) FROM {table_name} t WHERE NOT EXISTS ( SELECT 1 FROM {event_table_name} e - WHERE e.pgh_obj_id = t.id AND e.pgh_label = 'initial_import' + WHERE e.pgh_obj_id = t.id AND e.pgh_label = 'initial_backfill' ) """) backfill_count = cursor.fetchone()[0] @@ -165,7 +165,7 @@ def process_model_simple(self, model_name, batch_size, dry_run): if col == "pgh_created_at": select_columns.append("NOW() as pgh_created_at") elif col == "pgh_label": - select_columns.append("'initial_import' as pgh_label") + select_columns.append("'initial_backfill' as pgh_label") elif col == "pgh_obj_id": select_columns.append("t.id as pgh_obj_id") elif col == "pgh_context_id": @@ -181,7 +181,7 @@ def process_model_simple(self, model_name, batch_size, dry_run): SELECT t.id FROM {table_name} t WHERE NOT EXISTS ( SELECT 1 FROM {event_table_name} e - WHERE e.pgh_obj_id = t.id AND e.pgh_label = 'initial_import' + WHERE e.pgh_obj_id = t.id AND e.pgh_label = 'initial_backfill' ) ORDER BY t.id """) diff --git a/dojo/management/commands/pghistory_clear.py b/dojo/management/commands/pghistory_clear.py index a2593ac25ca..97b938293d7 100644 --- a/dojo/management/commands/pghistory_clear.py +++ b/dojo/management/commands/pghistory_clear.py @@ -35,11 +35,10 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - if not settings.ENABLE_AUDITLOG or settings.AUDITLOG_TYPE != "django-pghistory": + if not settings.ENABLE_AUDITLOG: self.stdout.write( self.style.WARNING( - "pghistory is not enabled. Set DD_ENABLE_AUDITLOG=True and " - "DD_AUDITLOG_TYPE=django-pghistory", + "pghistory is not enabled. Set DD_ENABLE_AUDITLOG=True", ), ) return diff --git a/dojo/management/commands/stamp_finding_last_reviewed.py b/dojo/management/commands/stamp_finding_last_reviewed.py deleted file mode 100644 index ade2bcb6bc3..00000000000 --- a/dojo/management/commands/stamp_finding_last_reviewed.py +++ /dev/null @@ -1,78 +0,0 @@ - -from auditlog.models import LogEntry -from django.contrib.contenttypes.models import ContentType -from django.core.management.base import BaseCommand - -from dojo.models import Finding - -""" -Authors: Jay Paz -New fields last_reviewed, last_reviewed_by, mitigated_by have been added to the Finding model -This script will update all findings with a last_reviewed date of the most current date from: -1. Finding Date if no other evidence of activity is found -2. Last note added date if a note is found -3. Mitigation Date if finding is mitigated -4. Last action_log entry date if Finding has been updated - -It will update the last_reviewed_by with the current reporter. - -If mitigated it will update the mitigated_by with last_reviewed_by or current reporter if last_reviewed_by is None -""" - - -class Command(BaseCommand): - help = ( - "A new field last_reviewed has been added to the Finding model \n" - "This script will update all findings with a last_reviewed date of the most current date from: \n" - "1. Finding Date if no other evidence of activity is found \n" - "2. Last note added date if a note is found \n" - "3. Mitigation Date if finding is mitigated \n" - "4. Last action_log entry date if Finding has been updated \n" - ) - - def handle(self, *args, **options): - findings = Finding.objects.all().order_by("id") - for finding in findings: - save = False - if not finding.last_reviewed: - date_discovered = finding.date - last_note_date = finding.date - - if finding.notes.all(): - last_note_date = finding.notes.order_by("-date")[ - 0].date.date() - - mitigation_date = finding.date - - if finding.mitigated: - mitigation_date = finding.mitigated.date() - - last_action_date = finding.date - - try: - ct = ContentType.objects.get_for_id( - ContentType.objects.get_for_model(finding).id) - obj = ct.get_object_for_this_type(pk=finding.id) - log_entries = LogEntry.objects.filter(content_type=ct, - object_pk=obj.id).order_by( - "-timestamp") - if log_entries: - last_action_date = log_entries[0].timestamp.date() - except KeyError: - pass - - finding.last_reviewed = max( - [date_discovered, last_note_date, mitigation_date, - last_action_date]) - save = True - - if not finding.last_reviewed_by: - finding.last_reviewed_by = finding.reporter - save = True - - if finding.mitigated: - if not finding.mitigated_by: - finding.mitigated_by = finding.last_reviewed_by or finding.reporter - save = True - if save: - finding.save() diff --git a/dojo/middleware.py b/dojo/middleware.py index 275cfa09e71..b23e94027af 100644 --- a/dojo/middleware.py +++ b/dojo/middleware.py @@ -7,15 +7,12 @@ import pghistory.middleware import requests -from auditlog.context import set_actor -from auditlog.middleware import AuditlogMiddleware as _AuditlogMiddleware from django.conf import settings from django.contrib import messages from django.db import models from django.http import HttpResponseRedirect from django.shortcuts import redirect from django.urls import reverse -from django.utils.functional import SimpleLazyObject from social_core.exceptions import AuthCanceled, AuthFailed, AuthForbidden, AuthTokenError from social_django.middleware import SocialAuthExceptionMiddleware from watson.middleware import SearchContextMiddleware @@ -117,6 +114,12 @@ def __init__(self, get_response): def __call__(self, request): self.load() try: + # Store error in request for context processor to display + # (We can't use messages here because MessageMiddleware runs after this middleware) + if hasattr(self._thread_local, "system_settings_error"): + request.system_settings_error = self._thread_local.system_settings_error + # Clear from thread-local after copying to request + delattr(self._thread_local, "system_settings_error") return self.get_response(request) finally: # ensure cleanup happens even if an exception occurs @@ -135,6 +138,8 @@ def get_system_settings(cls): def cleanup(cls, *args, **kwargs): # noqa: ARG003 if hasattr(cls._thread_local, "system_settings"): del cls._thread_local.system_settings + if hasattr(cls._thread_local, "system_settings_error"): + delattr(cls._thread_local, "system_settings_error") @classmethod def load(cls): @@ -150,13 +155,19 @@ class System_Settings_Manager(models.Manager): def get_from_db(self, *args, **kwargs): # logger.debug('refreshing system_settings from db') + from dojo.models import System_Settings # noqa: PLC0415 circular import try: from_db = super().get(*args, **kwargs) - except: - from dojo.models import System_Settings # noqa: PLC0415 circular import - # this mimics the existing code that was in filters.py and utils.py. - # cases I have seen triggering this is for example manage.py collectstatic inside a docker build where mysql is not available - # logger.debug('unable to get system_settings from database, constructing (new) default instance. Exception was:', exc_info=True) + except Exception as e: + # Store error message in thread-local for middleware to display + error_msg = str(e) + if hasattr(DojoSytemSettingsMiddleware._thread_local, "system_settings_error"): + # Only store the first error to avoid duplicates + pass + else: + DojoSytemSettingsMiddleware._thread_local.system_settings_error = error_msg + # Return defaults so app can still start - error will be displayed as warning message + # logger.debug('unable to get system_settings from database, returning defaults. Exception was:', exc_info=True) return System_Settings() return from_db @@ -211,20 +222,6 @@ def __call__(self, request): return self.get_response(request) -# This solution comes from https://github.com/jazzband/django-auditlog/issues/115#issuecomment-1539262735 -# It fix situation when TokenAuthentication is used in API. Otherwise, actor in AuditLog would be set to None -class AuditlogMiddleware(_AuditlogMiddleware): - def __call__(self, request): - remote_addr = self._get_remote_addr(request) - - user = SimpleLazyObject(lambda: getattr(request, "user", None)) - - context = set_actor(actor=user, remote_addr=remote_addr) - - with context: - return self.get_response(request) - - class PgHistoryMiddleware(pghistory.middleware.HistoryMiddleware): """ diff --git a/dojo/models.py b/dojo/models.py index 0628c33165d..57ce9c18e72 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -268,6 +268,8 @@ class UserContactInfo(models.Model): slack_user_id = models.CharField(blank=True, null=True, max_length=25) block_execution = models.BooleanField(default=False, help_text=_("Instead of async deduping a finding the findings will be deduped synchronously and will 'block' the user until completion.")) force_password_reset = models.BooleanField(default=False, help_text=_("Forces this user to reset their password on next login.")) + token_last_reset = models.DateTimeField(null=True, blank=True, help_text=_("Timestamp of the most recent API token reset for this user.")) + password_last_reset = models.DateTimeField(null=True, blank=True, help_text=_("Timestamp of the most recent password reset for this user.")) class Dojo_Group(models.Model): @@ -473,12 +475,6 @@ class System_Settings(models.Model): help_text=_("Enables Benchmarks such as the OWASP ASVS " "(Application Security Verification Standard)")) - enable_template_match = models.BooleanField( - default=False, - blank=False, - verbose_name=_("Enable Remediation Advice"), - help_text=_("Enables global remediation advice and matching on CWE and Title. The text will be replaced for mitigation, impact and references on a finding. Useful for providing consistent impact and remediation advice regardless of the scanner.")) - enable_similar_findings = models.BooleanField( default=True, blank=False, @@ -2817,10 +2813,6 @@ def save(self, dedupe_option=True, rules_option=True, product_grading_option=Tru self.set_hash_code(dedupe_option) if is_new_finding: - # We enter here during the first call from serializers.py - from dojo.utils import apply_cwe_to_template # noqa: PLC0415 circular import - # No need to use the returned variable since `self` Is updated in memory - apply_cwe_to_template(self) if (self.file_path is not None) and (len(self.unsaved_endpoints) == 0): self.static_finding = True self.dynamic_finding = False @@ -3656,6 +3648,9 @@ class Finding_Template(models.Model): verbose_name="Vulnerability Id", help_text="An id of a vulnerability in a security advisory associated with this finding. Can be a Common Vulnerabilities and Exposures (CVE) or from other sources.") cvssv3 = models.TextField(help_text=_("Common Vulnerability Scoring System version 3 (CVSSv3) score associated with this finding."), validators=[cvss3_validator], max_length=117, null=True, verbose_name=_("CVSS v3 vector")) + cvssv3_score = models.FloatField(null=True, blank=True, help_text=_("CVSSv3 score")) + cvssv4 = models.TextField(help_text=_("Common Vulnerability Scoring System version 4 (CVSS4) score associated with this finding."), validators=[cvss4_validator], max_length=255, null=True, verbose_name=_("CVSS4 vector")) + cvssv4_score = models.FloatField(null=True, blank=True, help_text=_("CVSSv4 score")) severity = models.CharField(max_length=200, null=True, blank=True) description = models.TextField(null=True, blank=True) @@ -3664,8 +3659,25 @@ class Finding_Template(models.Model): references = models.TextField(null=True, blank=True, db_column="refs") last_used = models.DateTimeField(null=True, editable=False) numerical_severity = models.CharField(max_length=4, null=True, blank=True, editable=False) - template_match = models.BooleanField(default=False, verbose_name=_("Template Match Enabled"), help_text=_("Enables this template for matching remediation advice. Match will be applied to all active, verified findings by CWE.")) - template_match_title = models.BooleanField(default=False, verbose_name=_("Match Template by Title and CWE"), help_text=_("Matches by title text (contains search) and CWE.")) + + # Remediation planning fields + fix_available = models.BooleanField(null=True, blank=True, help_text=_("Indicates if a fix is available for this vulnerability type")) + fix_version = models.CharField(max_length=100, null=True, blank=True, help_text=_("Version where fix is available")) + planned_remediation_version = models.CharField(max_length=99, null=True, blank=True, help_text=_("Target version for remediation")) + effort_for_fixing = models.CharField(max_length=99, null=True, blank=True, help_text=_("Effort estimate for fixing (e.g., Low/Medium/High)")) + + # Technical details fields + steps_to_reproduce = models.TextField(null=True, blank=True, help_text=_("Standard reproduction steps for this vulnerability type")) + severity_justification = models.TextField(null=True, blank=True, help_text=_("Explanation of why this severity level is appropriate")) + component_name = models.CharField(max_length=500, null=True, blank=True, help_text=_("Affected component name (e.g., library name)")) + component_version = models.CharField(max_length=100, null=True, blank=True, help_text=_("Affected component version")) + + # Notes field (single note content, not a list) + notes = models.TextField(null=True, blank=True, help_text=_("Note content to add when applying this template")) + + # String-based list fields (newline-separated) + vulnerability_ids_text = models.TextField(null=True, blank=True, help_text=_("Vulnerability IDs (one per line)")) + endpoints_text = models.TextField(null=True, blank=True, help_text=_("Endpoint URLs (one per line)")) tags = TagField(blank=True, force_lowercase=True, help_text=_("Add tags that help describe this finding template. Choose from the list or add new tags. Press Enter key to add.")) @@ -3685,16 +3697,20 @@ def get_breadcrumbs(self): return [{"title": str(self), "url": reverse("view_template", args=(self.id,))}] - @cached_property + @property def vulnerability_ids(self): - # Get vulnerability ids from database and convert to list of strings - vulnerability_ids_model = self.vulnerability_id_template_set.all() - vulnerability_ids = [vulnerability_id.vulnerability_id for vulnerability_id in vulnerability_ids_model] + """Parse vulnerability IDs from TextField string (newline-separated).""" + vulnerability_ids = [] - # Synchronize the cve field with the unsaved_vulnerability_ids + # Get from the TextField + if self.vulnerability_ids_text: + # Parse newline-separated string, remove empty lines + vulnerability_ids = [line.strip() for line in self.vulnerability_ids_text.split("\n") if line.strip()] + + # Synchronize the cve field with the vulnerability_ids # We do this to be as flexible as possible to handle the fields until # the cve field is not needed anymore and can be removed. - if vulnerability_ids and self.cve: + if vulnerability_ids and self.cve and self.cve not in vulnerability_ids: # Make sure the first entry of the list is the value of the cve field vulnerability_ids.insert(0, self.cve) elif not vulnerability_ids and self.cve: @@ -3704,10 +3720,13 @@ def vulnerability_ids(self): # Remove duplicates return list(dict.fromkeys(vulnerability_ids)) - -class Vulnerability_Id_Template(models.Model): - finding_template = models.ForeignKey(Finding_Template, editable=False, on_delete=models.CASCADE) - vulnerability_id = models.TextField(max_length=50, blank=False, null=False) + @property + def endpoints(self): + """Parse endpoint URLs from TextField string (newline-separated).""" + if not self.endpoints_text: + return [] + # Parse newline-separated string, remove empty lines + return [line.strip() for line in self.endpoints_text.split("\n") if line.strip()] class Check_List(models.Model): @@ -4862,7 +4881,6 @@ def __str__(self): admin.site.register(Development_Environment) admin.site.register(Finding_Template) admin.site.register(Vulnerability_Id) -admin.site.register(Vulnerability_Id_Template) admin.site.register(BurpRawRequestResponse) admin.site.register(Announcement) admin.site.register(UserAnnouncement) diff --git a/dojo/pghistory_models.py b/dojo/pghistory_models.py index 936bd939c60..0902c775ec1 100644 --- a/dojo/pghistory_models.py +++ b/dojo/pghistory_models.py @@ -22,10 +22,15 @@ class DojoEvents(pghistory.models.Events): as regular model fields instead of accessing nested JSON data. """ + # Middleware-provided fields user = pghistory.ProxyField("pgh_context__user", models.IntegerField(null=True)) url = pghistory.ProxyField("pgh_context__url", models.TextField(null=True)) remote_addr = pghistory.ProxyField("pgh_context__remote_addr", models.CharField(max_length=45, null=True)) + # Process identification fields + source = pghistory.ProxyField("pgh_context__source", models.CharField(max_length=50, null=True)) + scan_type = pghistory.ProxyField("pgh_context__scan_type", models.CharField(max_length=100, null=True)) + class Meta: proxy = True app_label = "dojo" diff --git a/dojo/pghistory_utils.py b/dojo/pghistory_utils.py new file mode 100644 index 00000000000..4477ee8c124 --- /dev/null +++ b/dojo/pghistory_utils.py @@ -0,0 +1,99 @@ +""" +Utilities for passing pghistory context to Celery tasks. + +pghistory uses thread-local storage, so context is lost when tasks run +in Celery workers. These utilities allow capturing context in the sender +process and recreating it in the worker. +""" +import uuid +from contextlib import nullcontext + +from pghistory import runtime as pghistory_runtime + + +def get_serializable_pghistory_context(): + """ + Capture the current pghistory context for passing to Celery tasks. + + Returns a JSON-serializable dict with context id and metadata, + or None if no context is active. + """ + if hasattr(pghistory_runtime._tracker, "value"): + ctx = pghistory_runtime._tracker.value + return { + "id": str(ctx.id), + "metadata": ctx.metadata.copy(), + } + return None + + +class PgHistoryContextFromTask: + + """ + Context manager to apply pghistory context received from a Celery task. + + This recreates the exact same context (with the same UUID) that was + active when the task was dispatched, ensuring all events share the + same pgh_context_id. + + Usage: + pgh_context = kwargs.pop("_pgh_context", None) + with PgHistoryContextFromTask(pgh_context): + # Task body runs here with context applied + """ + + def __init__(self, context_data): + """ + Initialize with context data from Celery kwargs. + + Args: + context_data: Dict with "id" (UUID string) and "metadata" (dict), + or None for no-op behavior. + + """ + self.context_data = context_data + self._pre_execute_hook = None + self._owns_context = False + + def __enter__(self): + if not self.context_data: + return None + + from django.db import connection # noqa: PLC0415 + + context_id = uuid.UUID(self.context_data["id"]) + metadata = self.context_data["metadata"] + + # Only create a new context if one doesn't already exist + if not hasattr(pghistory_runtime._tracker, "value"): + self._pre_execute_hook = connection.execute_wrapper( + pghistory_runtime._inject_history_context, + ) + self._pre_execute_hook.__enter__() + pghistory_runtime._tracker.value = pghistory_runtime.Context( + id=context_id, + metadata=metadata, + ) + self._owns_context = True + else: + # Context already exists, just merge metadata + pghistory_runtime._tracker.value.metadata.update(metadata) + + return pghistory_runtime._tracker.value + + def __exit__(self, *exc): + if self._owns_context and self._pre_execute_hook: + delattr(pghistory_runtime._tracker, "value") + self._pre_execute_hook.__exit__(*exc) + + +def get_pghistory_context_manager(context_data): + """ + Return appropriate context manager for the given context data. + + Returns PgHistoryContextFromTask if context_data is provided, + otherwise returns a no-op nullcontext. + """ + if context_data: + return PgHistoryContextFromTask(context_data) + return nullcontext() diff --git a/dojo/product/signals.py b/dojo/product/signals.py index 0ed9a62747c..efcf23da5aa 100644 --- a/dojo/product/signals.py +++ b/dojo/product/signals.py @@ -1,9 +1,7 @@ import contextlib -from auditlog.models import LogEntry from django.conf import settings from django.contrib.auth import get_user_model -from django.contrib.contenttypes.models import ContentType from django.db.models.signals import post_delete, post_save from django.dispatch import receiver from django.urls import reverse @@ -37,7 +35,7 @@ def product_post_delete(sender, instance, **kwargs): user = None if settings.ENABLE_AUDITLOG: - # First try to find deletion author in pghistory events + # Find deletion author in pghistory events # Look for delete events for this specific product instance pghistory_delete_events = DojoEvents.objects.filter( pgh_obj_model="dojo.Product", @@ -53,15 +51,7 @@ def product_post_delete(sender, instance, **kwargs): with contextlib.suppress(User.DoesNotExist): user = User.objects.get(id=latest_delete.user) - # Fall back to django-auditlog if no user found in pghistory - if not user: - if le := LogEntry.objects.filter( - action=LogEntry.Action.DELETE, - content_type=ContentType.objects.get(app_label="dojo", model="product"), - object_id=instance.id, - ).order_by("-id").first(): - user = le.actor - + # Fallback to the current user of the request (Which might be not available for ASYNC_OBJECT_DELETE scenario's) if not user: current_user = get_current_user() user = current_user diff --git a/dojo/product_type/signals.py b/dojo/product_type/signals.py index 523e7dcedc4..8bb435751d5 100644 --- a/dojo/product_type/signals.py +++ b/dojo/product_type/signals.py @@ -1,10 +1,8 @@ import contextlib -from auditlog.models import LogEntry from crum import get_current_user from django.conf import settings from django.contrib.auth import get_user_model -from django.contrib.contenttypes.models import ContentType from django.db.models.signals import post_delete, post_save from django.dispatch import receiver from django.urls import reverse @@ -37,7 +35,7 @@ def product_type_post_delete(sender, instance, **kwargs): user = None if settings.ENABLE_AUDITLOG: - # First try to find deletion author in pghistory events + # Find deletion author in pghistory events # Look for delete events for this specific product_type instance pghistory_delete_events = DojoEvents.objects.filter( pgh_obj_model="dojo.Product_Type", @@ -53,16 +51,6 @@ def product_type_post_delete(sender, instance, **kwargs): with contextlib.suppress(User.DoesNotExist): user = User.objects.get(id=latest_delete.user) - # Fall back to django-auditlog if no user found in pghistory - if not user: - if le := LogEntry.objects.filter( - action=LogEntry.Action.DELETE, - content_type=ContentType.objects.get(app_label="dojo", model="product_type"), - object_id=instance.id, - ).order_by("-id").first(): - user = le.actor - - # Since adding pghistory as auditlog option, this signal here runs before the django-auditlog signal # Fallback to the current user of the request (Which might be not available for ASYNC_OBJECT_DELETE scenario's) if not user: current_user = get_current_user() diff --git a/dojo/reports/views.py b/dojo/reports/views.py index ae6a99804eb..1dc29ff5e2d 100644 --- a/dojo/reports/views.py +++ b/dojo/reports/views.py @@ -7,6 +7,7 @@ from dateutil.relativedelta import relativedelta from django.conf import settings from django.core.exceptions import PermissionDenied +from django.db.models import Prefetch from django.http import Http404, HttpRequest, HttpResponse, QueryDict from django.shortcuts import get_object_or_404, render from django.utils import timezone @@ -28,7 +29,7 @@ from dojo.finding.views import BaseListFindings from dojo.forms import ReportOptionsForm from dojo.labels import get_labels -from dojo.models import Dojo_User, Endpoint, Engagement, Finding, Product, Product_Type, Test +from dojo.models import Dojo_User, Endpoint, Engagement, Finding, Notes, Product, Product_Type, Test from dojo.reports.widgets import ( CoverPage, CustomReportJsonForm, @@ -642,7 +643,7 @@ def prefetch_related_findings_for_report(findings): "burprawrequestresponse_set", "endpoints", "tags", - "notes", + Prefetch("notes", queryset=Notes.objects.filter(private=False)), "files", "reporter", "mitigated_by", @@ -815,6 +816,7 @@ def add_extra_values(self): def get(self, request): findings, _obj = get_findings(request) + findings = prefetch_related_findings_for_report(findings) self.findings = findings findings = self.add_findings_data() response = HttpResponse(content_type="text/csv") @@ -850,6 +852,8 @@ def get(self, request): "endpoints", "vulnerability_ids", "tags", + "status", + "notes", )) self.fields = fields self.add_extra_headers() @@ -913,6 +917,20 @@ def get(self, request): tags_value = tags_value.removesuffix("; ") fields.append(tags_value) + # Status + status_value = finding.status() + fields.append(status_value) + + # Notes + notes_value = "" + for note in finding.notes.filter(private=False): + note_entry = note.entry.replace("\n", " NEWLINE ").replace("\r", "") + notes_value += f"{note_entry}; " + notes_value = notes_value.removesuffix("; ") + if len(notes_value) > EXCEL_CHAR_LIMIT: + notes_value = notes_value[:EXCEL_CHAR_LIMIT - 3] + "..." + fields.append(notes_value) + self.fields = fields self.finding = finding self.add_extra_values() @@ -935,6 +953,7 @@ def add_extra_values(self): def get(self, request): findings, _obj = get_findings(request) + findings = prefetch_related_findings_for_report(findings) self.findings = findings findings = self.add_findings_data() workbook = Workbook() @@ -990,6 +1009,12 @@ def get(self, request): cell = worksheet.cell(row=row_num, column=col_num, value="tags") cell.font = font_bold col_num += 1 + cell = worksheet.cell(row=row_num, column=col_num, value="status") + cell.font = font_bold + col_num += 1 + cell = worksheet.cell(row=row_num, column=col_num, value="notes") + cell.font = font_bold + col_num += 1 self.row_num = row_num self.col_num = col_num self.add_extra_headers() @@ -1056,6 +1081,23 @@ def get(self, request): tags_value = tags_value.removesuffix("; \n") worksheet.cell(row=row_num, column=col_num, value=tags_value) col_num += 1 + + # Status + status_value = finding.status() + worksheet.cell(row=row_num, column=col_num, value=status_value) + col_num += 1 + + # Notes + notes_value = "" + for note in finding.notes.filter(private=False): + note_entry = note.entry.replace("\r", "") + notes_value += f"{note_entry}; \n" + notes_value = notes_value.removesuffix("; \n") + if len(notes_value) > EXCEL_CHAR_LIMIT: + notes_value = notes_value[:EXCEL_CHAR_LIMIT - 3] + "..." + worksheet.cell(row=row_num, column=col_num, value=notes_value) + col_num += 1 + self.col_num = col_num self.row_num = row_num self.finding = finding diff --git a/dojo/risk_acceptance/helper.py b/dojo/risk_acceptance/helper.py index 1aa09e82669..1ce170deabb 100644 --- a/dojo/risk_acceptance/helper.py +++ b/dojo/risk_acceptance/helper.py @@ -1,6 +1,7 @@ import logging from contextlib import suppress +import pghistory from dateutil.relativedelta import relativedelta from django.core.exceptions import PermissionDenied from django.urls import reverse @@ -175,39 +176,42 @@ def expiration_handler(*args, **kwargs): If configured also sends a JIRA comment in both case to each jira issue. This is per finding. """ - try: - system_settings = System_Settings.objects.get() - except System_Settings.DoesNotExist: - logger.warning("Unable to get system_settings, skipping risk acceptance expiration job") + # Wrap with pghistory context for audit trail + with pghistory.context(source="risk_acceptance_expiration"): + try: + system_settings = System_Settings.objects.get() + except System_Settings.DoesNotExist: + logger.warning("Unable to get system_settings, skipping risk acceptance expiration job") + return - risk_acceptances = get_expired_risk_acceptances_to_handle() + risk_acceptances = get_expired_risk_acceptances_to_handle() - logger.info("expiring %i risk acceptances that are past expiration date", len(risk_acceptances)) - for risk_acceptance in risk_acceptances: - expire_now(risk_acceptance) - # notification created by expire_now code + logger.info("expiring %i risk acceptances that are past expiration date", len(risk_acceptances)) + for risk_acceptance in risk_acceptances: + expire_now(risk_acceptance) + # notification created by expire_now code - heads_up_days = system_settings.risk_acceptance_notify_before_expiration - if heads_up_days > 0: - risk_acceptances = get_almost_expired_risk_acceptances_to_handle(heads_up_days) + heads_up_days = system_settings.risk_acceptance_notify_before_expiration + if heads_up_days > 0: + risk_acceptances = get_almost_expired_risk_acceptances_to_handle(heads_up_days) - logger.info("notifying for %i risk acceptances that are expiring within %i days", len(risk_acceptances), heads_up_days) - for risk_acceptance in risk_acceptances: - logger.debug("notifying for risk acceptance %i:%s with %i findings", risk_acceptance.id, risk_acceptance, len(risk_acceptance.accepted_findings.all())) + logger.info("notifying for %i risk acceptances that are expiring within %i days", len(risk_acceptances), heads_up_days) + for risk_acceptance in risk_acceptances: + logger.debug("notifying for risk acceptance %i:%s with %i findings", risk_acceptance.id, risk_acceptance, len(risk_acceptance.accepted_findings.all())) - notification_title = "Risk acceptance with " + str(len(risk_acceptance.accepted_findings.all())) + " accepted findings will expire on " + \ - timezone.localtime(risk_acceptance.expiration_date).strftime("%b %d, %Y") + " for " + \ - str(risk_acceptance.engagement.product) + ": " + str(risk_acceptance.engagement.name) + notification_title = "Risk acceptance with " + str(len(risk_acceptance.accepted_findings.all())) + " accepted findings will expire on " + \ + timezone.localtime(risk_acceptance.expiration_date).strftime("%b %d, %Y") + " for " + \ + str(risk_acceptance.engagement.product) + ": " + str(risk_acceptance.engagement.name) - create_notification(event="risk_acceptance_expiration", title=notification_title, risk_acceptance=risk_acceptance, - accepted_findings=risk_acceptance.accepted_findings.all(), engagement=risk_acceptance.engagement, - product=risk_acceptance.engagement.product, - url=reverse("view_risk_acceptance", args=(risk_acceptance.engagement.id, risk_acceptance.id))) + create_notification(event="risk_acceptance_expiration", title=notification_title, risk_acceptance=risk_acceptance, + accepted_findings=risk_acceptance.accepted_findings.all(), engagement=risk_acceptance.engagement, + product=risk_acceptance.engagement.product, + url=reverse("view_risk_acceptance", args=(risk_acceptance.engagement.id, risk_acceptance.id))) - post_jira_comments(risk_acceptance, risk_acceptance.accepted_findings.all(), expiration_warning_message_creator, heads_up_days) + post_jira_comments(risk_acceptance, risk_acceptance.accepted_findings.all(), expiration_warning_message_creator, heads_up_days) - risk_acceptance.expiration_date_warned = timezone.now() - risk_acceptance.save() + risk_acceptance.expiration_date_warned = timezone.now() + risk_acceptance.save() def get_view_risk_acceptance(risk_acceptance: Risk_Acceptance) -> str: diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 6798d66cfa1..ab7918c922c 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -31,7 +31,7 @@ DD_SITE_URL=(str, "http://localhost:8080"), DD_DEBUG=(bool, False), DD_DJANGO_DEBUG_TOOLBAR_ENABLED=(bool, False), - # django-auditlog imports django-jsonfield-backport raises a warning that can be ignored, + # django-jsonfield-backport raises a warning that can be ignored, # see https://github.com/laymonage/django-jsonfield-backport # debug_toolbar.E001 is raised when running tests in dev mode via run-unittests.sh DD_SILENCED_SYSTEM_CHECKS=(list, ["debug_toolbar.E001", "django_jsonfield_backport.W001"]), @@ -277,7 +277,6 @@ # regular expression to exclude one or more parsers # could be usefull to limit parser allowed # AWS Scout2 Scan Parser is deprecated (see https://github.com/DefectDojo/django-DefectDojo/pull/5268) - DD_PARSER_EXCLUDE=(str, ""), # when enabled in sytem settings, every minute a job run to delete excess duplicates # we limit the amount of duplicates that can be deleted in a single run of that job # to prevent overlapping runs of that job from occurrring @@ -286,6 +285,8 @@ DD_EDITABLE_MITIGATED_DATA=(bool, False), # new feature that tracks history across multiple reimports for the same test DD_TRACK_IMPORT_HISTORY=(bool, True), + # Batch size for reimport candidate matching (finding existing findings) + DD_IMPORT_REIMPORT_MATCH_BATCH_SIZE=(int, 1000), # Batch size for import/reimport deduplication processing DD_IMPORT_REIMPORT_DEDUPE_BATCH_SIZE=(int, 1000), # Delete Auditlogs older than x month; -1 to keep all logs @@ -343,12 +344,9 @@ DD_DEDUPLICATION_ALGORITHM_PER_PARSER=(str, ""), # Dictates whether cloud banner is created or not DD_CREATE_CLOUD_BANNER=(bool, True), - # With this setting turned on, Dojo maintains an audit log of changes made to entities (Findings, Tests, Engagements, Procuts, ...) - # If you run big import you may want to disable this because the way django-auditlog currently works, there's - # a big performance hit. Especially during (re-)imports. + # With this setting turned on, Dojo maintains an audit log of changes made to entities (Findings, Tests, Engagements, Products, ...) + # If you run big import you may want to disable this because there's a performance hit during (re-)imports. DD_ENABLE_AUDITLOG=(bool, True), - # Audit logging system: "django-auditlog" (default) or "django-pghistory" - DD_AUDITLOG_TYPE=(str, "django-auditlog"), # Specifies whether the "first seen" date of a given report should be used over the "last seen" date DD_USE_FIRST_SEEN=(bool, False), # When set to True, use the older version of the qualys parser that is a more heavy handed in setting severity @@ -717,6 +715,7 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param SIMILAR_FINDINGS_MAX_RESULTS = env("DD_SIMILAR_FINDINGS_MAX_RESULTS") MAX_REQRESP_FROM_API = env("DD_MAX_REQRESP_FROM_API") MAX_AUTOCOMPLETE_WORDS = env("DD_MAX_AUTOCOMPLETE_WORDS") +ENABLE_AUDITLOG = env("DD_ENABLE_AUDITLOG") LOGIN_EXEMPT_URLS = ( rf"^{URL_PREFIX}static/", @@ -974,7 +973,6 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param "dojo.middleware.AdditionalHeaderMiddleware", "dojo.middleware.CustomSocialAuthExceptionMiddleware", "crum.CurrentRequestUserMiddleware", - "dojo.middleware.AuditlogMiddleware", "dojo.middleware.AsyncSearchContextMiddleware", "dojo.request_cache.middleware.RequestCacheMiddleware", "dojo.middleware.LongRunningRequestAlertMiddleware", @@ -982,6 +980,15 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param MIDDLEWARE = DJANGO_MIDDLEWARE_CLASSES +if ENABLE_AUDITLOG: + middleware_list = list(MIDDLEWARE) + crum_index = middleware_list.index("crum.CurrentRequestUserMiddleware") + + # Insert pghistory HistoryMiddleware before CurrentRequestUserMiddleware + middleware_list.insert(crum_index, "dojo.middleware.PgHistoryMiddleware") + + MIDDLEWARE = middleware_list + # WhiteNoise allows your web app to serve its own static files, # making it a self-contained unit that can be deployed anywhere without relying on nginx if env("DD_WHITENOISE"): @@ -1713,6 +1720,7 @@ def saml2_attrib_map_format(din): DISABLE_FINDING_MERGE = env("DD_DISABLE_FINDING_MERGE") TRACK_IMPORT_HISTORY = env("DD_TRACK_IMPORT_HISTORY") +IMPORT_REIMPORT_MATCH_BATCH_SIZE = env("DD_IMPORT_REIMPORT_MATCH_BATCH_SIZE") IMPORT_REIMPORT_DEDUPE_BATCH_SIZE = env("DD_IMPORT_REIMPORT_DEDUPE_BATCH_SIZE") # ------------------------------------------------------------------------------ @@ -1855,9 +1863,6 @@ def saml2_attrib_map_format(din): # If using this, lines for Qualys WAS deduplication functions must be un-commented QUALYS_WAS_UNIQUE_ID = False -# exclusion list for parsers -PARSER_EXCLUDE = env("DD_PARSER_EXCLUDE") - SERIALIZATION_MODULES = { "xml": "tagulous.serializers.xml_serializer", "json": "tagulous.serializers.json", @@ -1998,10 +2003,6 @@ def saml2_attrib_map_format(din): # Auditlog # ------------------------------------------------------------------------------ AUDITLOG_FLUSH_RETENTION_PERIOD = env("DD_AUDITLOG_FLUSH_RETENTION_PERIOD") -ENABLE_AUDITLOG = env("DD_ENABLE_AUDITLOG") -AUDITLOG_TYPE = env("DD_AUDITLOG_TYPE") -AUDITLOG_TWO_STEP_MIGRATION = False -AUDITLOG_USE_TEXT_CHANGES_IF_JSON_IS_NOT_PRESENT = False AUDITLOG_FLUSH_BATCH_SIZE = env("DD_AUDITLOG_FLUSH_BATCH_SIZE") AUDITLOG_FLUSH_MAX_BATCHES = env("DD_AUDITLOG_FLUSH_MAX_BATCHES") @@ -2093,19 +2094,6 @@ def show_toolbar(request): # Auditlog configuration # ######################################################################################################### -if ENABLE_AUDITLOG: - middleware_list = list(MIDDLEWARE) - crum_index = middleware_list.index("crum.CurrentRequestUserMiddleware") - - if AUDITLOG_TYPE == "django-auditlog": - # Insert AuditlogMiddleware before CurrentRequestUserMiddleware - middleware_list.insert(crum_index, "dojo.middleware.AuditlogMiddleware") - elif AUDITLOG_TYPE == "django-pghistory": - # Insert pghistory HistoryMiddleware before CurrentRequestUserMiddleware - middleware_list.insert(crum_index, "dojo.middleware.PgHistoryMiddleware") - - MIDDLEWARE = middleware_list - PGHISTORY_FOREIGN_KEY_FIELD = pghistory.ForeignKey(db_index=False) PGHISTORY_CONTEXT_FIELD = pghistory.ContextForeignKey(db_index=True) PGHISTORY_OBJ_FIELD = pghistory.ObjForeignKey(db_index=True) diff --git a/dojo/tasks.py b/dojo/tasks.py index f74ffbd6389..d02040fa5b3 100644 --- a/dojo/tasks.py +++ b/dojo/tasks.py @@ -1,6 +1,7 @@ import logging from datetime import timedelta +import pghistory from celery.utils.log import get_task_logger from django.apps import apps from django.conf import settings @@ -99,6 +100,13 @@ def flush_auditlog(*args, **kwargs): @app.task(bind=True) def async_dupe_delete(*args, **kwargs): + # Wrap with pghistory context for audit trail + with pghistory.context(source="dupe_delete_task"): + _async_dupe_delete_impl() + + +def _async_dupe_delete_impl(): + """Internal implementation of async_dupe_delete within pghistory context.""" try: system_settings = System_Settings.objects.get() enabled = system_settings.delete_duplicates @@ -182,12 +190,19 @@ def async_sla_compute_and_notify_task(*args, **kwargs): @app.task def jira_status_reconciliation_task(*args, **kwargs): - return jira_status_reconciliation(*args, **kwargs) + # Wrap with pghistory context for audit trail + with pghistory.context( + source="jira_reconciliation", + mode=kwargs.get("mode", "reconcile"), + ): + return jira_status_reconciliation(*args, **kwargs) @app.task def fix_loop_duplicates_task(*args, **kwargs): - return fix_loop_duplicates() + # Wrap with pghistory context for audit trail + with pghistory.context(source="fix_loop_duplicates"): + return fix_loop_duplicates() @app.task diff --git a/dojo/templates/dojo/action_history.html b/dojo/templates/dojo/action_history.html index f3867024943..904347b7c11 100644 --- a/dojo/templates/dojo/action_history.html +++ b/dojo/templates/dojo/action_history.html @@ -70,8 +70,8 @@

{{ h.pgh_obj_id|default:"N/A" }} - {% if h.pgh_label == "initial_import" %} - Initial Import + {% if h.pgh_label == "initial_backfill" %} + Initial Backfill {% elif h.pgh_diff %}
{% for field, values in h.pgh_diff.items %} diff --git a/dojo/templates/dojo/add_findings.html b/dojo/templates/dojo/add_findings.html index 465e547be26..9a552cb0fd6 100644 --- a/dojo/templates/dojo/add_findings.html +++ b/dojo/templates/dojo/add_findings.html @@ -23,7 +23,7 @@

Add Findings to a Test

{% if temp %} -
+ {% csrf_token %} {% include "dojo/form_fields.html" with form=form %} {% if jform %} diff --git a/dojo/templates/dojo/add_template.html b/dojo/templates/dojo/add_template.html index 541d2999f10..3953559891b 100644 --- a/dojo/templates/dojo/add_template.html +++ b/dojo/templates/dojo/add_template.html @@ -22,11 +22,6 @@

{{ name }} {{ template }}

{% csrf_token %} {% include "dojo/form_fields.html" with form=form %}
- {% if count > 0 %} -
-

{{ count }} active, verified finding{{ count|pluralize }} match this template.

-
- {% endif %}
diff --git a/dojo/templates/dojo/templates.html b/dojo/templates/dojo/templates.html index 12cec47befc..0abe328aa00 100644 --- a/dojo/templates/dojo/templates.html +++ b/dojo/templates/dojo/templates.html @@ -57,7 +57,6 @@

{% comment %} The display field is translated in the function. No need to translate here as well{% endcomment %} {% dojo_sort request 'CWE' 'cwe' 'asc' %} - Match Enabled {% comment %} The display field is translated in the function. No need to translate here as well{% endcomment %} {% dojo_sort request 'Name' 'title' %} {% comment %} The display field is translated in the function. No need to translate here as well{% endcomment %} @@ -76,10 +75,9 @@

{% endif %} - {{ finding.template_match }} {% if add_from_template %} - @@ -110,7 +108,7 @@

{% else %}

{% endif %}{{ finding.severity }}

{% if add_from_template %} - Use This Template diff --git a/dojo/templates/dojo/users.html b/dojo/templates/dojo/users.html index e05b7346915..dedbcc2c8cb 100644 --- a/dojo/templates/dojo/users.html +++ b/dojo/templates/dojo/users.html @@ -69,6 +69,8 @@

{% trans "Global Role" %} {% trans "Date Joined" %} {% trans "Last Login" %} + {% trans "Token Last Reset" %} + {% trans "Password Last Reset" %} {% block users_table_extra_header_rows %} {% endblock users_table_extra_header_rows %} @@ -127,6 +129,8 @@

{% if u.global_role.role %} {{ u.global_role.role }} {% endif %} {{ u.date_joined }} {% if u.last_login %}{{ u.last_login }}{% else %}{% trans "Never" %}{% endif %} + {% if u.usercontactinfo.token_last_reset %}{{ u.usercontactinfo.token_last_reset }}{% else %}{% trans "Never" %}{% endif %} + {% if u.usercontactinfo.password_last_reset %}{{ u.usercontactinfo.password_last_reset }}{% else %}{% trans "Never" %}{% endif %} {% block users_table_extra_data_rows %} {% endblock users_table_extra_data_rows %} diff --git a/dojo/templates/dojo/view_finding.html b/dojo/templates/dojo/view_finding.html index 2626130ed07..b03f632c6b5 100755 --- a/dojo/templates/dojo/view_finding.html +++ b/dojo/templates/dojo/view_finding.html @@ -101,24 +101,7 @@

{% endif %} - {% if cwe_template.cwe and finding|has_object_permission:"Finding_Edit" %} -
  • - - Apply CWE Template Remediation to Finding - -
    • - - {% csrf_token %} - - - {% endif %} - {% if not cwe_template.cwe and "Finding_Add"|has_global_permission %} -
  • - - Create a CWE Remediation Template - -
    • - {% endif %} + {% if finding|has_object_permission:"Finding_Edit" %}
  • @@ -1273,12 +1256,6 @@

    Credential }); - $('a.apply-cwe-finding').on('click', function (e) { - if (confirm('Apply CWE template to this finding? (This will overwrite mitigation, impact and references.)')) { - $("form#apply-cwe-finding-form").submit(); - } - - }); $("a#next").on('click', function (e) { e.preventDefault(); diff --git a/dojo/templates/dojo/view_user.html b/dojo/templates/dojo/view_user.html index 74a377e8999..7c9876572ab 100644 --- a/dojo/templates/dojo/view_user.html +++ b/dojo/templates/dojo/view_user.html @@ -34,7 +34,7 @@

    {% trans "Default Information" %}

    {% if "auth.delete_user"|has_configuration_permission:request and user.id != request.user.id %}
  • - + {% trans "Delete" %}
    • {% endif %} @@ -374,6 +374,14 @@

    {% trans "Last Login" %} {% if user.last_login %} {{ user.last_login }} {% else %} {% trans "Never" %} {% endif %} + + {% trans "Token Last Reset" %} + {% if user.usercontactinfo.token_last_reset %} {{ user.usercontactinfo.token_last_reset }} {% else %} {% trans "Never" %} {% endif %} + + + {% trans "Password Last Reset" %} + {% if user.usercontactinfo.password_last_reset %} {{ user.usercontactinfo.password_last_reset }} {% else %} {% trans "Never" %} {% endif %} +

    @@ -399,11 +407,11 @@

    {% if field.view_codename %} - {% endif %} diff --git a/dojo/test/signals.py b/dojo/test/signals.py index 22669a2a040..cc180c5d55e 100644 --- a/dojo/test/signals.py +++ b/dojo/test/signals.py @@ -1,9 +1,7 @@ import contextlib -from auditlog.models import LogEntry from django.conf import settings from django.contrib.auth import get_user_model -from django.contrib.contenttypes.models import ContentType from django.db.models.signals import post_delete, pre_delete, pre_save from django.dispatch import receiver from django.urls import reverse @@ -25,7 +23,7 @@ def test_post_delete(sender, instance, using, origin, **kwargs): user = None if settings.ENABLE_AUDITLOG: - # First try to find deletion author in pghistory events + # Find deletion author in pghistory events # Look for delete events for this specific test instance pghistory_delete_events = DojoEvents.objects.filter( pgh_obj_model="dojo.Test", @@ -41,15 +39,6 @@ def test_post_delete(sender, instance, using, origin, **kwargs): with contextlib.suppress(User.DoesNotExist): user = User.objects.get(id=latest_delete.user) - # Fall back to django-auditlog if no user found in pghistory - if not user: - if le := LogEntry.objects.filter( - action=LogEntry.Action.DELETE, - content_type=ContentType.objects.get(app_label="dojo", model="test"), - object_id=instance.id, - ).order_by("-id").first(): - user = le.actor - # Update description with user if found if user: description = _('The test "%(name)s" was deleted by %(user)s') % { diff --git a/dojo/test/urls.py b/dojo/test/urls.py index 6bea2675615..335cf260b86 100644 --- a/dojo/test/urls.py +++ b/dojo/test/urls.py @@ -23,7 +23,7 @@ views.AddFindingView.as_view(), name="add_findings"), re_path(r"^test/(?P\d+)/add_findings/(?P\d+)$", - views.add_temp_finding, name="add_temp_finding"), + views.add_finding_from_template, name="add_finding_from_template"), re_path(r"^test/(?P\d+)/search$", views.search, name="search"), re_path( r"^test/(?P\d+)/re_import_scan_results", diff --git a/dojo/test/views.py b/dojo/test/views.py index d2bf11092e7..a05c0b3b660 100644 --- a/dojo/test/views.py +++ b/dojo/test/views.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta from functools import reduce +import pghistory from django.contrib import messages from django.contrib.admin.utils import NestedObjects from django.core.exceptions import ValidationError @@ -658,12 +659,12 @@ def post(self, request: HttpRequest, test_id: int): @user_is_authorized(Test, Permissions.Finding_Add, "tid") -def add_temp_finding(request, tid, fid): +def add_finding_from_template(request, tid, fid): jform = None test = get_object_or_404(Test, id=tid) - finding = get_object_or_404(Finding_Template, id=fid) + template = get_object_or_404(Finding_Template, id=fid) findings = Finding_Template.objects.all() - push_all_jira_issues = jira_helper.is_push_all_issues(finding) + push_all_jira_issues = jira_helper.is_push_all_issues(template) if request.method == "POST": @@ -690,8 +691,8 @@ def add_temp_finding(request, tid, fid): _("Can not set a finding as inactive or false positive without adding all mandatory notes"), extra_tags="alert-danger") if form.is_valid(): - finding.last_used = timezone.now() - finding.save() + template.last_used = timezone.now() + template.save() new_finding = form.save(commit=False) new_finding.test = test new_finding.reporter = request.user @@ -699,14 +700,23 @@ def add_temp_finding(request, tid, fid): new_finding.severity) new_finding.tags = form.cleaned_data["tags"] - new_finding.cvssv3 = finding.cvssv3 new_finding.date = form.cleaned_data["date"] or datetime.today() finding_helper.update_finding_status(new_finding, request.user) new_finding.save(dedupe_option=False) - # Save and add new endpoints + # Copy all fields from template + finding_helper.copy_template_fields_to_finding( + finding=new_finding, + template=template, + user=request.user, + copy_vulnerability_ids=True, + copy_endpoints=True, + copy_notes=True, + ) + + # Save and add new endpoints from form (user may have added more) finding_helper.add_endpoints(new_finding, form) new_finding.save() @@ -737,20 +747,64 @@ def add_temp_finding(request, tid, fid): extra_tags="alert-danger") else: - form = AddFindingForm(req_resp=None, product=test.engagement.product, initial={"active": False, - "date": timezone.now().date(), - "verified": False, - "false_p": False, - "duplicate": False, - "out_of_scope": False, - "title": finding.title, - "description": finding.description, - "cwe": finding.cwe, - "severity": finding.severity, - "mitigation": finding.mitigation, - "impact": finding.impact, - "references": finding.references, - "numerical_severity": finding.numerical_severity}) + # Build initial data with all template fields + initial_data = { + "active": False, + "date": timezone.now().date(), + "verified": False, + "false_p": False, + "duplicate": False, + "out_of_scope": False, + "title": template.title, + "description": template.description, + "cwe": template.cwe, + "severity": template.severity, + "mitigation": template.mitigation, + "impact": template.impact, + "references": template.references, + "numerical_severity": template.numerical_severity, + } + + # Add CVSS fields + if template.cvssv3: + initial_data["cvssv3"] = template.cvssv3 + if template.cvssv3_score is not None: + initial_data["cvssv3_score"] = template.cvssv3_score + if template.cvssv4: + initial_data["cvssv4"] = template.cvssv4 + if template.cvssv4_score is not None: + initial_data["cvssv4_score"] = template.cvssv4_score + + # Add remediation fields + if template.fix_available is not None: + initial_data["fix_available"] = template.fix_available + if template.fix_version: + initial_data["fix_version"] = template.fix_version + if template.planned_remediation_version: + initial_data["planned_remediation_version"] = template.planned_remediation_version + if template.effort_for_fixing: + initial_data["effort_for_fixing"] = template.effort_for_fixing + + # Add technical details fields + if template.steps_to_reproduce: + initial_data["steps_to_reproduce"] = template.steps_to_reproduce + if template.severity_justification: + initial_data["severity_justification"] = template.severity_justification + if template.component_name: + initial_data["component_name"] = template.component_name + if template.component_version: + initial_data["component_version"] = template.component_version + + # Add vulnerability IDs + if template.vulnerability_ids: + initial_data["vulnerability_ids"] = " ".join(template.vulnerability_ids) + + # Add endpoints to endpoints_to_add field + if template.endpoints: + endpoint_urls = template.endpoints if isinstance(template.endpoints, list) else template.endpoints.split("\n") + initial_data["endpoints_to_add"] = "\n".join([url.strip() for url in endpoint_urls if url.strip()]) + + form = AddFindingForm(req_resp=None, product=test.engagement.product, initial=initial_data) if jira_helper.get_jira_project(test): jform = JIRAFindingForm(push_all=jira_helper.is_push_all_issues(test), prefix="jiraform", jira_project=jira_helper.get_jira_project(test), finding_form=form) @@ -763,7 +817,7 @@ def add_temp_finding(request, tid, fid): "jform": jform, "findings": findings, "temp": True, - "fid": finding.id, + "fid": template.id, "tid": test.id, "test": test, }) @@ -1075,6 +1129,12 @@ def post( if form_error := self.process_form(request, context.get("form"), context): add_error_message_to_response(form_error) return self.failure_redirect(request, context) + # Add pghistory context for audit trail (adds to existing middleware context) + pghistory.context( + source="reimport", + test_id=context.get("test").id, + scan_type=context.get("scan_type"), + ) # Kick off the import process if import_error := self.reimport_findings(context): add_error_message_to_response(import_error) diff --git a/dojo/tools/checkmarx/parser.py b/dojo/tools/checkmarx/parser.py index 71ca3e8074c..3cdf42f5560 100644 --- a/dojo/tools/checkmarx/parser.py +++ b/dojo/tools/checkmarx/parser.py @@ -40,7 +40,7 @@ def get_fields(self) -> list[str]: - component_version: Set to value within the "name" returned from the Checkmarx Scanner. """ return [ - "title" + "title", "cwe", "active", "verified", diff --git a/dojo/tools/dependency_check/parser.py b/dojo/tools/dependency_check/parser.py index ec14ac8f196..d6f524325d3 100644 --- a/dojo/tools/dependency_check/parser.py +++ b/dojo/tools/dependency_check/parser.py @@ -87,7 +87,7 @@ def add_finding(self, finding, dupes): def get_filename_and_path_from_dependency( self, dependency, related_dependency, namespace, ): - if not related_dependency: + if related_dependency is None: return dependency.findtext( f"{namespace}fileName", ), dependency.findtext(f"{namespace}filePath") @@ -105,10 +105,10 @@ def get_component_name_and_version_from_dependency( self, dependency, related_dependency, namespace, ): identifiers_node = dependency.find(namespace + "identifiers") - if identifiers_node: + if identifiers_node is not None: # analyzing identifier from the more generic to package_node = identifiers_node.find(".//" + namespace + "package") - if package_node: + if package_node is not None: pck_id = package_node.findtext(f"{namespace}id") purl = PackageURL.from_string(pck_id) purl_parts = purl.to_dict() @@ -166,7 +166,7 @@ def get_component_name_and_version_from_dependency( maven_node = identifiers_node.find( ".//" + namespace + 'identifier[@type="maven"]', ) - if maven_node: + if maven_node is not None: maven_parts = maven_node.findtext(f"{namespace}name").split( ":", ) @@ -181,7 +181,7 @@ def get_component_name_and_version_from_dependency( evidence_collected_node = dependency.find( namespace + "evidenceCollected", ) - if evidence_collected_node: + if evidence_collected_node is not None: # # # file @@ -199,12 +199,12 @@ def get_component_name_and_version_from_dependency( product_node = evidence_collected_node.find( ".//" + namespace + 'evidence[@type="product"]', ) - if product_node: + if product_node is not None: component_name = product_node.findtext(f"{namespace}value") version_node = evidence_collected_node.find( ".//" + namespace + 'evidence[@type="version"]', ) - if version_node: + if version_node is not None: component_version = version_node.findtext( f"{namespace}value", ) @@ -280,7 +280,7 @@ def get_finding_from_vulnerability( mitigated = None is_Mitigated = False name = vulnerability.findtext(f"{namespace}name") - if vulnerability.find(f"{namespace}cwes"): + if vulnerability.find(f"{namespace}cwes") is not None: cwe_field = vulnerability.find(f"{namespace}cwes").findtext( f"{namespace}cwe", ) @@ -425,14 +425,14 @@ def get_findings(self, filename, test): dependencies = scan.find(namespace + "dependencies") scan_date = None - if scan.find(f"{namespace}projectInfo"): + if scan.find(f"{namespace}projectInfo") is not None: projectInfo_node = scan.find(f"{namespace}projectInfo") if projectInfo_node.findtext(f"{namespace}reportDate"): scan_date = dateutil.parser.parse( projectInfo_node.findtext(f"{namespace}reportDate"), ) - if dependencies: + if dependencies is not None: for dependency in dependencies.findall(namespace + "dependency"): vulnerabilities = dependency.find( namespace + "vulnerabilities", @@ -441,7 +441,7 @@ def get_findings(self, filename, test): for vulnerability in vulnerabilities.findall( namespace + "vulnerability", ): - if vulnerability: + if vulnerability is not None: finding = self.get_finding_from_vulnerability( dependency, None, @@ -456,7 +456,7 @@ def get_findings(self, filename, test): relatedDependencies = dependency.find( namespace + "relatedDependencies", ) - if relatedDependencies: + if relatedDependencies is not None: for ( relatedDependency ) in relatedDependencies.findall( @@ -479,7 +479,7 @@ def get_findings(self, filename, test): for suppressedVulnerability in vulnerabilities.findall( namespace + "suppressedVulnerability", ): - if suppressedVulnerability: + if suppressedVulnerability is not None: finding = self.get_finding_from_vulnerability( dependency, None, diff --git a/dojo/tools/factory.py b/dojo/tools/factory.py index a536607f640..dfdd887d065 100644 --- a/dojo/tools/factory.py +++ b/dojo/tools/factory.py @@ -6,8 +6,6 @@ from inspect import isclass from pathlib import Path -from django.conf import settings - from dojo.models import Test_Type, Tool_Configuration, Tool_Type PARSERS = {} @@ -37,12 +35,12 @@ def get_parser(scan_type): if scan_type not in PARSERS: msg = f"Parser '{scan_type}' does not exist" raise ValueError(msg) - rg = re.compile(settings.PARSER_EXCLUDE) - if not rg.match(scan_type) or not settings.PARSER_EXCLUDE.strip(): - # update DB dynamically - test_type, _ = Test_Type.objects.get_or_create(name=scan_type) - if test_type.active: - return PARSERS[scan_type] + + # update DB dynamically + test_type, _ = Test_Type.objects.get_or_create(name=scan_type) + if test_type.active: + return PARSERS[scan_type] + msg = f"Parser {scan_type} is not active" raise ValueError(msg) diff --git a/dojo/tools/fortify/xml_parser.py b/dojo/tools/fortify/xml_parser.py index ce86719e7c1..f04b55db3c4 100644 --- a/dojo/tools/fortify/xml_parser.py +++ b/dojo/tools/fortify/xml_parser.py @@ -86,7 +86,7 @@ def xml_structure_before_24_2(self, root, test): for group in ReportSection.iter("GroupingSection"): title = group.findtext("groupTitle") maj_attr_summary = group.find("MajorAttributeSummary") - if maj_attr_summary: + if maj_attr_summary is not None: meta_info = maj_attr_summary.findall("MetaInfo") meta_pair[place][title] = { x.findtext("Name"): x.findtext("Value") @@ -115,11 +115,11 @@ def xml_structure_before_24_2(self, root, test): "FilePath": issue.find("Primary").find("FilePath").text, "LineStart": issue.find("Primary").find("LineStart").text, } - if issue.find("Primary").find("Snippet"): + if issue.find("Primary").find("Snippet") is not None: details["Snippet"] = issue.find("Primary").find("Snippet").text else: details["Snippet"] = "n/a" - if issue.find("Source"): + if issue.find("Source") is not None: source = { "FileName": issue.find("Source").find("FileName").text, "FilePath": issue.find("Source").find("FilePath").text, diff --git a/dojo/tools/nmap/parser.py b/dojo/tools/nmap/parser.py index a402d421e65..0f8af861df0 100644 --- a/dojo/tools/nmap/parser.py +++ b/dojo/tools/nmap/parser.py @@ -96,7 +96,8 @@ def get_findings(self, file, test): ) description += service_info script_id = None - if script := port_element.find("script"): + script = port_element.find("script") + if script is not None: if script_id := script.attrib.get("id"): description += f"**Script ID:** {script_id}\n" if script_output := script.attrib.get("output"): diff --git a/dojo/tools/pingcastle/__init__.py b/dojo/tools/pingcastle/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dojo/tools/pingcastle/parser.py b/dojo/tools/pingcastle/parser.py new file mode 100644 index 00000000000..e346184a875 --- /dev/null +++ b/dojo/tools/pingcastle/parser.py @@ -0,0 +1,285 @@ + +import contextlib +import datetime +import re + +from defusedxml.ElementTree import parse + +from dojo.models import Endpoint, Finding + + +class PingCastleParser: + + CVE_REGEX = re.compile(r"(CVE-\d{4}-\d{4,7})", re.IGNORECASE) + + _SEVERITY_ORDER = ["Info", "Low", "Medium", "High", "Critical"] + + def get_scan_types(self): + return ["PingCastle"] + + def get_label_for_scan_types(self, scan_type): + return scan_type + + def get_description_for_scan_types(self, scan_type): + return "PingCastle XML export" + + def get_findings(self, file, test): + try: + tree = parse(file) + root = tree.getroot() + except Exception as e: + exception = f"Invalid PingCastle XML format: {e}" + raise ValueError(exception) + dupes = {} + report_date = self._parse_datetime(root.findtext("GenerationDate")) + domain_fqdn = root.findtext("DomainFQDN") or "" + dc_infos, dc_endpoints = self._collect_domain_controllers(root) + findings = [] + for rr in root.findall("RiskRules/HealthcheckRiskRule"): + points = self._safe_int(rr.findtext("Points")) + category = rr.findtext("Category") or "" + model = rr.findtext("Model") or "" + risk_id = rr.findtext("RiskId") or "" + rationale = rr.findtext("Rationale") or "" + severity = self._map_points_to_severity(points) + severity = self._apply_contextual_bump( + severity=severity, + category=category, + model=model, + risk_id=risk_id, + rationale=rationale, + ) + if not severity or severity not in self._SEVERITY_ORDER: + severity = "Info" + title = f"[PingCastle] {risk_id} ({category}/{model})" + description = self._compose_risk_rule_description( + domain_fqdn=domain_fqdn, + risk_id=risk_id, + points=points, + category=category, + model=model, + rationale=rationale, + dc_infos=dc_infos, + root=root, + ) + finding = Finding( + title=title, + test=test, + description=description, + severity=severity, + mitigation="Review and remediate according to PingCastle recommendations.", + impact="Risk identified by PingCastle HealthCheck.", + vuln_id_from_tool=risk_id, + ) + if report_date: + finding.date = report_date + cves = list(self.CVE_REGEX.findall(rationale or "")) + if cves: + finding.unsaved_vulnerability_ids = cves + finding.unsaved_endpoints = [] + if self._is_dc_specific_risk(risk_id, model, rationale): + finding.unsaved_endpoints.extend(dc_endpoints) + elif domain_fqdn: + finding.unsaved_endpoints.append(Endpoint(host=domain_fqdn)) + if risk_id == "A-DC-Coerce": + self._enrich_coerce_with_rpc_interfaces(finding, dc_infos) + if risk_id == "A-DC-Spooler": + self._enrich_spooler_status(finding, dc_infos) + if risk_id == "A-MinPwdLen": + self._enrich_password_policy(finding, root) + dupe_key = risk_id + if dupe_key in dupes: + existing = dupes[dupe_key] + existing.description += "\n\n-----\n\n" + finding.description + existing.unsaved_endpoints.extend(finding.unsaved_endpoints) + else: + dupes[dupe_key] = finding + findings.extend(list(dupes.values())) + return findings + + def _compose_risk_rule_description( + self, + domain_fqdn, + risk_id, + points, + category, + model, + rationale, + dc_infos, + root, + ): + lines = [] + lines.append("### PingCastle Risk Rule") # noqa: FURB113 + lines.append(f"**Domain**: `{domain_fqdn}`") + lines.append(f"**RiskId**: `{risk_id}`") + lines.append(f"**Category/Model**: `{category}` / `{model}`") + lines.append(f"**Points**: `{points}`") + if rationale: + lines.append(f"**Rationale**: {rationale}") + if risk_id.startswith("A-DC-") or "DomainControllers" in root.tag: + if dc_infos: + lines.append("\n#### Domain Controllers") + for dc in dc_infos: + ips = ", ".join(dc.get("ips", [])) + lines.append( + f"- **{dc['name']}** (OS: {dc.get('os', '?')}, IPs: {ips}, " + f"SpoolerRemote: {dc.get('remote_spooler', 'false')})", + ) + return "\n".join(lines) + + def _collect_domain_controllers(self, root): + dc_infos = [] + endpoints = [] + for dc in root.findall("DomainControllers/HealthcheckDomainController"): + name = dc.findtext("DCName") or "" + os = dc.findtext("OperatingSystem") or "" + remote_spooler = dc.findtext("RemoteSpoolerDetected") or "false" + ip_elems = dc.findall("IP/string") + ips = [ip_elem.text for ip_elem in ip_elems if ip_elem is not None and ip_elem.text] + dc_info = { + "name": name, + "os": os, + "remote_spooler": remote_spooler.lower() == "true", + "ips": ips, + "rpc_interfaces": [], + } + for rpc in dc.findall("RPCInterfacesOpen/HealthcheckDCRPCInterface"): + dc_info["rpc_interfaces"].append({ + "ip": rpc.attrib.get("IP", ""), + "interface": rpc.attrib.get("Interface", ""), + "opnum": rpc.attrib.get("OpNum", ""), + "function": rpc.attrib.get("Function", ""), + }) + dc_infos.append(dc_info) + if name: + endpoints.append(Endpoint(host=name)) + endpoints.extend(Endpoint(host=ip) for ip in ips) + return dc_infos, endpoints + + def _enrich_coerce_with_rpc_interfaces(self, finding, dc_infos): + added_any = False + for dc in dc_infos: + if dc.get("rpc_interfaces"): + if not added_any: + finding.description += "\n\n#### RPC Interfaces (potential coercion surface)\n" + added_any = True + finding.description += f"\n**{dc['name']}**:\n" + for ri in dc["rpc_interfaces"]: + finding.description += ( + f"- IP: `{ri['ip']}` | Interface: `{ri['interface']}` | " + f"OpNum: `{ri['opnum']}` | Function: `{ri['function']}`\n" + ) + + def _enrich_spooler_status(self, finding, dc_infos): + any_remote_spooler = any(dc.get("remote_spooler") for dc in dc_infos) + finding.description += ( + f"\n\n**Remote spooler exposure detected**: `{any_remote_spooler}`" + ) + + def _enrich_password_policy(self, finding, root): + min_len = None + complexity = None + for prop in root.findall("GPPPasswordPolicy/GPPSecurityPolicy/Properties/GPPSecurityPolicyProperty"): + key = (prop.findtext("Property") or "").strip() + val = (prop.findtext("Value") or "").strip() + if key == "MinimumPasswordLength": + min_len = val + elif key == "PasswordComplexity": + complexity = val + if min_len is not None or complexity is not None: + finding.description += "\n\n#### Observed Password Policy from GPO\n" + if min_len is not None: + finding.description += f"- MinimumPasswordLength: `{min_len}`\n" + if complexity is not None: + friendly = {"0": "disabled", "1": "enabled"}.get(complexity, complexity) + finding.description += f"- PasswordComplexity: `{friendly}`\n" + + @staticmethod + def _parse_datetime(text): + if not text: + return None + with contextlib.suppress(ValueError): + return datetime.datetime.fromisoformat(text) + return None + + @staticmethod + def _safe_int(text): + try: + return int(text) + except (TypeError, ValueError): + return 0 + + @staticmethod + def _map_points_to_severity(points): + if points <= 0: + return "Info" + if points <= 5: + return "Low" + if points <= 10: + return "Medium" + if points <= 15: + return "High" + return "Critical" + + @staticmethod + def _is_dc_specific_risk(risk_id: str, model: str = "", rationale: str = "") -> bool: + """ + Best effort classification: return True if the risk targets Domain Controllers specifically. + Signals: + - RiskId prefixes for DC: "A-DC-" (anomalies on DC), "S-DC-" (stale/DC subnet), and known IDs. + - Model contains DC-specific notions (e.g., "Audit" with RiskId A-AuditDC). + - Rationale text mentions DC count/context ("from X DC", "on domain controllers"). + """ + rid = (risk_id or "").strip() + mod = (model or "").strip() + rat = (rationale or "").strip().lower() + dc_prefixes = ("A-DC-", "S-DC-") + if rid.startswith(dc_prefixes): + return True + dc_specific_ids = { + "A-DC-Spooler", + "A-DC-Coerce", + "A-AuditDC", + "S-DC-SubnetMissing", + } + if rid in dc_specific_ids: + return True + if mod == "Audit" and rid.endswith("DC"): + return True + dc_markers = ( + " from ", + " dc", + " dcs", + " domain controller", + " domain controllers", + ) + return bool(any(marker in rat for marker in dc_markers)) + + def _apply_contextual_bump(self, severity: str, category: str = "", model: str = "", + risk_id: str = "", rationale: str = "") -> str: + """ + Minimal additive logic on top of points-based severity: + - If a CVE is mentioned -> bump by 1 level (at least Low). + If rationale indicates missing/not enabled mitigation -> ensure at least Medium. + - If DC-specific -> bump by 1 level. + - If category is 'Exposure' -> bump by 1 level. + """ + if not severity or severity not in self._SEVERITY_ORDER: + severity = "Info" + idx = self._SEVERITY_ORDER.index(severity) + rat = (rationale or "").lower() + cat = (category or "").strip().lower() + + if self.CVE_REGEX.search(rationale or ""): + idx = min(idx + 1, len(self._SEVERITY_ORDER) - 1) + mitigation_markers = ("mitigation", "not set", "disabled", "missing", "not enabled", "enable") + if any(m in rat for m in mitigation_markers): + idx = max(idx, self._SEVERITY_ORDER.index("Medium")) + + if self._is_dc_specific_risk(risk_id, model, rationale): + idx = min(idx + 1, len(self._SEVERITY_ORDER) - 1) + + if cat == "exposure": + idx = min(idx + 1, len(self._SEVERITY_ORDER) - 1) + + return self._SEVERITY_ORDER[idx] diff --git a/dojo/tools/prowler/__init__.py b/dojo/tools/prowler/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dojo/tools/prowler/parser.py b/dojo/tools/prowler/parser.py new file mode 100644 index 00000000000..f211c3e4c18 --- /dev/null +++ b/dojo/tools/prowler/parser.py @@ -0,0 +1,24 @@ +from dojo.tools.prowler.parser_csv import ProwlerParserCSV +from dojo.tools.prowler.parser_json import ProwlerParserJSON + + +class ProwlerParser: + + """Prowler is an Open Cloud Security that automates security and compliance in cloud environments. This parser is for Prowler JSON files and Prowler CSV files.""" + + def get_scan_types(self): + return ["Prowler Scan"] + + def get_label_for_scan_types(self, scan_type): + return "Prowler Scan" + + def get_description_for_scan_types(self, scan_type): + return "Prowler report file can be imported in JSON format or in CSV format." + + def get_findings(self, filename, test): + name = getattr(filename, "name", str(filename)).lower() + if name.endswith(".csv"): + return ProwlerParserCSV().get_findings(filename, test) + if name.endswith(".json"): + return ProwlerParserJSON().get_findings(filename, test) + return [] diff --git a/dojo/tools/prowler/parser_csv.py b/dojo/tools/prowler/parser_csv.py new file mode 100644 index 00000000000..0a9e286050d --- /dev/null +++ b/dojo/tools/prowler/parser_csv.py @@ -0,0 +1,122 @@ +import csv +import hashlib +import io + +from dojo.models import Finding + + +class ProwlerParserCSV: + + """Parser for Prowler CSV (semicolon-separated).""" + + def get_findings(self, filename, test): + if filename is None: + return [] + + content = filename.read() + if isinstance(content, bytes): + content = content.decode("utf-8") + reader = csv.DictReader(io.StringIO(content), delimiter=";") + csvarray = [] + for row in reader: + csvarray.append(row) + + dupes = {} + for row in csvarray: + # Skip vulnerability if the status is "PASS", continue parsing is status is "FAIL" or "MANUAL" + if row.get("STATUS") == "PASS": + continue + + provider = row.get("PROVIDER", "N/A").upper() + + description = ( + "**Cloud Type** : " + + provider + + "\n\n" + + "**Description** : " + + row.get("DESCRIPTION", "N/A") + + "\n\n" + + "**Service Name** : " + + row.get("SERVICE_NAME", "N/A") + + "\n\n" + + "**Status Detail** : " + + row.get("STATUS_EXTENDED", "N/A") + + "\n\n" + + "**Finding Created Time** : " + + row.get("TIMESTAMP", "N/A") + + "\n\n" + + "**Region** : " + + row.get("REGION", "N/A") + + "\n\n" + + "**Notes** : " + + row.get("NOTES", "N/A") + ) + + related = row.get("RELATED_URL", "") + additional = row.get("ADDITIONAL_URLS", "") + if related: + description += "\n\n**Related URL** : " + related + if additional: + description += "\n\n**Additional URLs** : " + additional + + mitigation = ( + "**Remediation Recommendation** : " + + row.get("REMEDIATION_RECOMMENDATION_TEXT", "N/A") + + "\n\n" + + "**Remediation Recommendation URL** : " + + row.get("REMEDIATION_RECOMMENDATION_URL", "N/A") + + "\n\n" + + "**Remediation Code Native IaC** : " + + row.get("REMEDIATION_CODE_NATIVEIAC", "N/A") + + "\n\n" + + "**Remediation Code Terraform** : " + + row.get("REMEDIATION_CODE_TERRAFORM", "N/A") + + "\n\n" + + "**Remediation Code CLI** : " + + row.get("REMEDIATION_CODE_CLI", "N/A") + + "\n\n" + + "**Other Remediation Info** : " + + row.get("REMEDIATION_CODE_OTHER", "N/A") + ) + + title = row.get("CHECK_TITLE", "") + severity = self.convert_severity(row.get("SEVERITY")) + impact = row.get("RISK", "") + compliance = row.get("COMPLIANCE", "N/A") + + # If compliance is not 'N/A', break info into multiple lines + references = "\n".join(part.strip() for part in compliance.split("|")) if compliance != "N/A" else "N/A" + + finding = Finding( + title=title, + test=test, + description=description, + severity=severity, + references=references, + mitigation=mitigation, + impact=impact, + static_finding=False, + dynamic_finding=True, + ) + + key = hashlib.sha256((finding.title + "|" + finding.description).encode("utf-8")).hexdigest() + if key not in dupes: + dupes[key] = finding + + return list(dupes.values()) + + def convert_severity(self, severity: str) -> str: + """Convert severity value""" + if not severity: + return "Info" + + s = severity.lower() + if s == "critical": + return "Critical" + if s == "high": + return "High" + if s == "medium": + return "Medium" + if s == "low": + return "Low" + return "Info" diff --git a/dojo/tools/prowler/parser_json.py b/dojo/tools/prowler/parser_json.py new file mode 100644 index 00000000000..82a62f9ba43 --- /dev/null +++ b/dojo/tools/prowler/parser_json.py @@ -0,0 +1,137 @@ +import hashlib +import json + +from dojo.models import Finding + + +class ProwlerParserJSON: + + """This parser is for Prowler JSON files.""" + + def get_findings(self, file, test): + data = json.load(file) + + dupes = {} + for node in data: + # Skip vulnerability if the status is "PASS", continue parsing is status is "FAIL" or "MANUAL" + if node.get("status_code") == "PASS": + continue + + cloudtype = self.get_cloud_type(node) + description = ( + "**Cloud Type** : " + + cloudtype + + "\n\n" + + "**Finding Description** : " + + node.get("finding_info", {}).get("desc", "N/A") + + "\n\n" + + "**Product Name** : " + + node.get("metadata", {}).get("product", {}).get("name", "N/A") + + "\n\n" + + "**Status Detail** : " + + node.get("status_detail", "N/A") + + "\n\n" + + "**Finding Created Time** : " + + node.get("finding_info", {}).get("created_time_dt", "N/A") + ) + # Add cloud type sepecific information to description + description = self.add_cloud_type_metadata(node, cloudtype, description) + + title = node.get("message", "") + severity = self.convert_severity(node.get("severity")) + mitigation = ( + "**Remediation Description** : " + + node.get("remediation", {}).get("desc", "N/A") + + "\n\n" + + "**Remediation References** : " + + ", ".join(node.get("remediation", {}).get("references", [])) + ) + impact = node.get("risk_details", "") + compliance = node.get("unmapped", {}).get("compliance", {}) + references = "**Related URL** : " + node.get("unmapped", {}).get("related_url", "") + # Add data presnet in scan to References + for key, values in compliance.items(): + joined = ", ".join(values) + # Ex: CIS-1.10 : 1.2.16 + references += f"\n\n**{key}** : {joined}" + + finding = Finding( + title=title, + test=test, + description=description, + severity=severity, + references=references, + mitigation=mitigation, + impact=impact, + static_finding=False, + dynamic_finding=True, + ) + + # internal de-duplication + dupe_key = hashlib.sha256(str(description + title).encode("utf-8")).hexdigest() + if dupe_key in dupes: + find = dupes[dupe_key] + if finding.description: + find.description += "\n" + finding.description + # find.unsaved_endpoints.extend(finding.unsaved_endpoints) + dupes[dupe_key] = find + else: + dupes[dupe_key] = finding + + return list(dupes.values()) + + def convert_severity(self, severity: str) -> str: + """Convert severity value""" + if not severity: + return "Info" + + s = severity.lower() + if s == "critical": + return "Critical" + if s == "high": + return "High" + if s == "medium": + return "Medium" + if s == "low": + return "Low" + return "Info" + + def get_cloud_type(self, node: dict) -> str: + """Determine the cloud type of a Prowler JSON finding. Returns one of: AWS, Azure, Kubernetes, GCP, or N/A""" + # Check for GCP, AWS, or Azure + account_type = node.get("cloud", {}).get("provider") + if account_type: + account_type.lower() + if account_type == "gcp": + return "GCP" + if account_type == "aws": + return "AWS" + if account_type == "azure": + return "AZURE" + + # Check for Kubernetes + for resource in node.get("resources", []): + namespace = resource.get("data", {}).get("metadata", {}).get("namespace") + if namespace is not None: + return "KUBERNETES" + + # No Cloud Type information was found + return "N/A" + + def add_cloud_type_metadata(self, node: dict, cloudtype: str, description: str) -> str: + # Add metadata for GCP, AWS, and Azure + if cloudtype in {"GCP", "AWS", "AZURE"}: + description += "\n\n" + "**" + cloudtype + " Region** : " + node.get("cloud", {}).get("region", "N/A") + return description + + # Add metadata for Kubernetes + if cloudtype == "KUBERNETES": + for resource in node.get("resources", []): + pod = resource.get("data", {}).get("metadata", {}).get("name") + namespace = resource.get("data", {}).get("metadata", {}).get("namespace") + if pod is not None: + description += "\n\n" + "**Pod Name** : " + pod + if namespace is not None: + description += "\n\n" + "**Namespace** : " + namespace + return description + return description diff --git a/dojo/tools/scout_suite/parser.py b/dojo/tools/scout_suite/parser.py index 36a63888ea9..f3671466f34 100644 --- a/dojo/tools/scout_suite/parser.py +++ b/dojo/tools/scout_suite/parser.py @@ -157,10 +157,10 @@ def recursive_print(self, src, depth=0, key=""): def tabs(n): return " " * n * 2 if isinstance(src, dict): - for _key, value in src.items(): + for dictkey, value in src.items(): if isinstance(src, str): - self.item_data = self.item_data + _key + "\n" - self.recursive_print(value, depth + 1, _key) + self.item_data = self.item_data + dictkey + "\n" + self.recursive_print(value, depth + 1, dictkey) elif isinstance(src, list): for litem in src: self.recursive_print(litem, depth + 2) diff --git a/dojo/tools/snyk/parser.py b/dojo/tools/snyk/parser.py index a3cdd43cb1e..f67092153ba 100644 --- a/dojo/tools/snyk/parser.py +++ b/dojo/tools/snyk/parser.py @@ -40,7 +40,7 @@ def get_fields(self) -> list[str]: "severity_justification", "description", "mitigation", - "component_name" + "component_name", "component_version", "false_p", "duplicate", diff --git a/dojo/tools/tool_issue_updater.py b/dojo/tools/tool_issue_updater.py index add5beb54a5..fd203edebea 100644 --- a/dojo/tools/tool_issue_updater.py +++ b/dojo/tools/tool_issue_updater.py @@ -1,3 +1,5 @@ +import pghistory + from dojo.celery import app from dojo.decorators import dojo_async_task, dojo_model_from_id, dojo_model_to_id from dojo.tools.api_sonarqube.parser import SCAN_SONARQUBE_API @@ -30,8 +32,9 @@ def tool_issue_updater(finding, *args, **kwargs): @dojo_async_task @app.task def update_findings_from_source_issues(**kwargs): + # Wrap with pghistory context for audit trail + with pghistory.context(source="sonarqube_sync"): + findings = SonarQubeApiUpdaterFromSource().get_findings_to_update() - findings = SonarQubeApiUpdaterFromSource().get_findings_to_update() - - for finding in findings: - SonarQubeApiUpdaterFromSource().update(finding) + for finding in findings: + SonarQubeApiUpdaterFromSource().update(finding) diff --git a/dojo/tools/wiz/parser.py b/dojo/tools/wiz/parser.py index 41a22c3c616..909471311c3 100644 --- a/dojo/tools/wiz/parser.py +++ b/dojo/tools/wiz/parser.py @@ -54,7 +54,7 @@ def parse_findings(self, test: Test, reader: csv.DictReader) -> list[Finding]: "Container Service", "Provider ID", "Risks", - "Threats" + "Threats", "Created At", "Status Changed At", "Updated At", diff --git a/dojo/user/authentication.py b/dojo/user/authentication.py new file mode 100644 index 00000000000..11f8fce104c --- /dev/null +++ b/dojo/user/authentication.py @@ -0,0 +1,59 @@ +from django.conf import settings +from django.urls import reverse +from django.utils import timezone +from rest_framework.authtoken.models import Token +from rest_framework.exceptions import PermissionDenied, ValidationError + +from dojo.authorization.authorization import user_is_superuser_or_global_owner +from dojo.models import Dojo_User, UserContactInfo +from dojo.notifications.helper import create_notification + + +def reset_token_for_user(*, acting_user: Dojo_User, target_user: Dojo_User, allow_self_reset: bool = False) -> None: + if not settings.API_TOKENS_ENABLED: + msg = "API tokens are disabled." + raise PermissionDenied(msg) + + if acting_user is None or getattr(acting_user, "is_anonymous", False): + msg = "Authentication required." + raise PermissionDenied(msg) + + if acting_user == target_user and not allow_self_reset: + msg = "Resetting your own API token via this endpoint is not allowed." + raise ValidationError(msg) + + # Only check permissions if not self-reset (self-reset is always allowed when allow_self_reset=True) + if acting_user != target_user and not user_is_superuser_or_global_owner(acting_user): + msg = "Insufficient permissions to reset API tokens." + raise PermissionDenied(msg) + + # Rotate token: delete existing token (if any), then create a new one. + Token.objects.filter(user=target_user).delete() + Token.objects.create(user=target_user) + + uci, _ = UserContactInfo.objects.get_or_create(user=target_user) + uci.token_last_reset = timezone.now() + uci.save(update_fields=["token_last_reset"]) + + # Send notification to the target user + if acting_user == target_user: + # Self-reset notification + description = f"A new API token has been generated for user {target_user.username}." + requested_by = None + else: + # Admin reset notification + description = ( + f"Your API token has been reset by {acting_user.get_full_name() or acting_user.username}. " + f"Please retrieve the new API token via the UI to keep using the API." + ) + requested_by = acting_user + + create_notification( + event="other", + title="API Token Reset", + description=description, + recipients=[target_user], + url=reverse("api_v2_key"), + requested_by=requested_by, + icon="key", + ) diff --git a/dojo/user/urls.py b/dojo/user/urls.py index 01971b808d1..d560bdb5f55 100644 --- a/dojo/user/urls.py +++ b/dojo/user/urls.py @@ -35,7 +35,7 @@ re_path(r"^password_reset/done/$", auth_views.PasswordResetDoneView.as_view( template_name="login/password_reset_done.html", ), name="password_reset_done"), - re_path(r"^reset/(?P[0-9A-Za-z_\-]+)/(?P[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,40})/$", auth_views.PasswordResetConfirmView.as_view( + re_path(r"^reset/(?P[0-9A-Za-z_\-]+)/(?P[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,40})/$", views.DojoPasswordResetConfirmView.as_view( template_name="login/password_reset_confirm.html", ), name="password_reset_confirm"), re_path(r"^reset/done/$", auth_views.PasswordResetCompleteView.as_view( diff --git a/dojo/user/views.py b/dojo/user/views.py index f4e2539d659..dd732b5c91e 100644 --- a/dojo/user/views.py +++ b/dojo/user/views.py @@ -10,7 +10,7 @@ from django.contrib.auth import logout from django.contrib.auth.decorators import login_required, user_passes_test from django.contrib.auth.forms import AuthenticationForm, PasswordResetForm -from django.contrib.auth.views import LoginView, PasswordResetView +from django.contrib.auth.views import LoginView, PasswordResetConfirmView, PasswordResetView from django.contrib.humanize.templatetags.humanize import naturaltime from django.core import serializers from django.core.exceptions import PermissionDenied, ValidationError @@ -26,7 +26,10 @@ from django.utils.timezone import now from django.utils.translation import gettext as _ from rest_framework.authtoken.models import Token +from rest_framework.exceptions import PermissionDenied as RFPermissionDenied +from rest_framework.exceptions import ValidationError as RFValidationError +from dojo.authorization.authorization import user_is_superuser_or_global_owner from dojo.authorization.authorization_decorators import user_is_configuration_authorized from dojo.authorization.roles_permissions import Permissions from dojo.decorators import dojo_ratelimit @@ -47,9 +50,10 @@ ) from dojo.group.queries import get_authorized_group_members_for_user from dojo.labels import get_labels -from dojo.models import Alerts, Dojo_Group_Member, Dojo_User, Product_Member, Product_Type_Member +from dojo.models import Alerts, Dojo_Group_Member, Dojo_User, Product_Member, Product_Type_Member, UserContactInfo from dojo.product.queries import get_authorized_product_members_for_user from dojo.product_type.queries import get_authorized_product_type_members_for_user +from dojo.user.authentication import reset_token_for_user from dojo.utils import add_breadcrumb, get_page_items, get_setting, get_system_setting logger = logging.getLogger(__name__) @@ -91,15 +95,17 @@ def api_v2_key(request): form = APIKeyForm(request.POST, instance=request.user) if form.is_valid() and form.cleaned_data["id"] == request.user.id: try: - api_key = Token.objects.get(user=request.user) - api_key.delete() - api_key = Token.objects.create(user=request.user) - except Token.DoesNotExist: - api_key = Token.objects.create(user=request.user) - messages.add_message(request, - messages.SUCCESS, - _("API Key generated successfully."), - extra_tags="alert-success") + reset_token_for_user(acting_user=request.user, target_user=request.user, allow_self_reset=True) + except (RFPermissionDenied, RFValidationError) as e: + messages.add_message(request, + messages.ERROR, + _("API Key generation failed: %s") % str(e), + extra_tags="alert-danger") + else: + messages.add_message(request, + messages.SUCCESS, + _("API Key generated successfully."), + extra_tags="alert-success") else: raise PermissionDenied else: @@ -289,6 +295,12 @@ def change_password(request): user.set_password(new_password) user.disable_force_password_reset() user.save() + # Case: user is logged in and changes their password via the profile UI. + # We stamp password_last_reset here so this flow is tracked independently from + # the "forgot password" reset flow (handled in DojoPasswordResetConfirmView). + uci, _created = UserContactInfo.objects.get_or_create(user=user) + uci.password_last_reset = now() + uci.save(update_fields=["password_last_reset"]) messages.add_message(request, messages.SUCCESS, @@ -433,10 +445,37 @@ def edit_user(request, uid): global_role = global_role_form.save(commit=False) global_role.user = user global_role.save() + + # Handle API token reset if checkbox is checked + # Only allow superusers or global owners to reset tokens + token_reset_success = False + if user_is_superuser_or_global_owner(request.user): + reset_token = contact_form.cleaned_data.get("reset_api_token", False) + if reset_token: + try: + reset_token_for_user(acting_user=request.user, target_user=user) + token_reset_success = True + messages.add_message(request, + messages.SUCCESS, + _("API token reset successfully."), + extra_tags="alert-success") + except (RFPermissionDenied, RFValidationError) as e: + # If permission denied or validation error, log but don't fail the user save + messages.add_message(request, + messages.WARNING, + _("User saved successfully, but API token reset failed: %s") % str(e), + extra_tags="alert-warning") + messages.add_message(request, messages.SUCCESS, _("User saved successfully."), extra_tags="alert-success") + + # Re-instantiate forms to uncheck the checkbox after successful save + if token_reset_success: + # Reload contact from database to get updated token_last_reset timestamp + contact.refresh_from_db() + contact_form = UserContactInfoForm(instance=contact, user=user) else: messages.add_message(request, messages.ERROR, @@ -669,3 +708,16 @@ class DojoPasswordResetView(PasswordResetView): class DojoForgotUsernameView(PasswordResetView): form_class = DojoForgotUsernameForm + + +class DojoPasswordResetConfirmView(PasswordResetConfirmView): + def form_valid(self, form): + response = super().form_valid(form) + # Flow: user resets password via the emailed "forgot password" link. + # This uses PasswordResetConfirmView, so we stamp password_last_reset here + # because this flow does not pass through change_password(). + user = form.user + uci, _created = UserContactInfo.objects.get_or_create(user=user) + uci.password_last_reset = now() + uci.save(update_fields=["password_last_reset"]) + return response diff --git a/dojo/utils.py b/dojo/utils.py index f2cc71e8098..33e99846b81 100644 --- a/dojo/utils.py +++ b/dojo/utils.py @@ -12,6 +12,7 @@ from calendar import monthrange from collections.abc import Callable from datetime import date, datetime, timedelta +from functools import cached_property from math import pi, sqrt from pathlib import Path @@ -71,6 +72,7 @@ Product, System_Settings, Test, + Test_Type, User, ) from dojo.notifications.helper import create_notification @@ -86,6 +88,11 @@ """ +def get_visible_scan_types(): + """Returns a QuerySet of active Test_Type objects.""" + return Test_Type.objects.filter(active=True) + + def do_false_positive_history(finding, *args, **kwargs): """ Replicate false positives across product. @@ -1300,52 +1307,77 @@ def get_celery_worker_status(): # Used to display the counts and enabled tabs in the product view +# Uses @cached_property for lazy loading to avoid expensive queries on every page load +# See: https://github.com/DefectDojo/django-DefectDojo/issues/10313 class Product_Tab: def __init__(self, product, title=None, tab=None): - self.product = product - self.title = title - self.tab = tab - self.engagement_count = Engagement.objects.filter( - product=self.product, active=True).count() - self.open_findings_count = Finding.objects.filter(test__engagement__product=self.product, - false_p=False, - duplicate=False, - out_of_scope=False, - active=True, - mitigated__isnull=True).count() - active_endpoints = Endpoint.objects.filter( - product=self.product, + self._product = product + self._title = title + self._tab = tab + self._engagement = None + + @cached_property + def engagement_count(self): + return Engagement.objects.filter( + product=self._product, active=True).count() + + @cached_property + def open_findings_count(self): + return Finding.objects.filter( + test__engagement__product=self._product, + false_p=False, + duplicate=False, + out_of_scope=False, + active=True, + mitigated__isnull=True).count() + + @cached_property + def _active_endpoints(self): + return Endpoint.objects.filter( + product=self._product, status_endpoint__mitigated=False, status_endpoint__false_positive=False, status_endpoint__out_of_scope=False, status_endpoint__risk_accepted=False, ) - self.endpoints_count = active_endpoints.distinct().count() - self.endpoint_hosts_count = active_endpoints.values("host").distinct().count() - self.benchmark_type = Benchmark_Type.objects.filter( + + @cached_property + def endpoints_count(self): + return self._active_endpoints.distinct().count() + + @cached_property + def endpoint_hosts_count(self): + return self._active_endpoints.values("host").distinct().count() + + @cached_property + def benchmark_type(self): + return Benchmark_Type.objects.filter( enabled=True).order_by("name") - self.engagement = None def setTab(self, tab): - self.tab = tab + self._tab = tab def setEngagement(self, engagement): - self.engagement = engagement + self._engagement = engagement + @property def engagement(self): - return self.engagement + return self._engagement + @property def tab(self): - return self.tab + return self._tab def setTitle(self, title): - self.title = title + self._title = title + @property def title(self): - return self.title + return self._title + @property def product(self): - return self.product + return self._product def engagements(self): return self.engagement_count @@ -1359,9 +1391,6 @@ def endpoints(self): def endpoint_hosts(self): return self.endpoint_hosts_count - def benchmark_type(self): - return self.benchmark_type - # Used to display the counts and enabled tabs in the product view def tab_view_count(product_id): @@ -1398,27 +1427,6 @@ def add_language(product, language, files=1, code=1): pass -# Apply finding template data by matching CWE + Title or CWE -def apply_cwe_to_template(finding, *, override=False): - if System_Settings.objects.get().enable_template_match or override: - # Attempt to match on CWE and Title First - template = Finding_Template.objects.filter( - cwe=finding.cwe, title__icontains=finding.title, template_match=True).first() - - # If none then match on CWE - template = Finding_Template.objects.filter( - cwe=finding.cwe, template_match=True).first() - - if template: - finding.mitigation = template.mitigation - finding.impact = template.impact - finding.references = template.references - template.last_used = timezone.now() - template.save() - - return finding - - def truncate_with_dots(the_string, max_length_including_dots): if not the_string: return the_string diff --git a/dojo/views.py b/dojo/views.py index ae2303e3ea4..d6b33f8c08b 100644 --- a/dojo/views.py +++ b/dojo/views.py @@ -135,7 +135,7 @@ def action_history(request, cid, oid): else: product_tab.setEngagement(object_value.engagement) - # Get audit history from both systems separately + # Get audit history from pghistory (and legacy django-auditlog entries if available) auditlog_history = [] pghistory_history = [] diff --git a/helm/defectdojo/Chart.lock b/helm/defectdojo/Chart.lock index 1d693b06484..ecf2eea23ec 100644 --- a/helm/defectdojo/Chart.lock +++ b/helm/defectdojo/Chart.lock @@ -4,6 +4,6 @@ dependencies: version: 16.7.27 - name: valkey repository: oci://registry-1.docker.io/cloudpirates - version: 0.10.2 -digest: sha256:ffaad5072467386baa7b7142dde88375c007fd72f6bd7e97008107d1fc37751d -generated: "2025-12-15T10:35:03.656421-06:00" + version: 0.13.0 +digest: sha256:2ea1c1bef68a7e7fd2dee4262506c880e48c9703cb7ba9908d91b891fc630048 +generated: "2025-12-15T17:19:18.528341139Z" diff --git a/helm/defectdojo/Chart.yaml b/helm/defectdojo/Chart.yaml index 68c97d8282c..a39152718f0 100644 --- a/helm/defectdojo/Chart.yaml +++ b/helm/defectdojo/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 -appVersion: "2.53.5" +appVersion: "2.54.0" description: A Helm chart for Kubernetes to install DefectDojo name: defectdojo -version: 1.9.5 +version: 1.9.6 icon: https://defectdojo.com/hubfs/DefectDojo_favicon.png maintainers: - name: madchap @@ -14,7 +14,7 @@ dependencies: repository: "oci://us-docker.pkg.dev/os-public-container-registry/defectdojo" condition: postgresql.enabled - name: valkey - version: 0.10.2 + version: 0.13.0 repository: "oci://registry-1.docker.io/cloudpirates" condition: valkey.enabled # For correct syntax, check https://artifacthub.io/docs/topics/annotations/helm/ @@ -34,4 +34,4 @@ dependencies: # description: Critical bug annotations: artifacthub.io/prerelease: "false" - artifacthub.io/changes: "- kind: changed\n description: Bump DefectDojo to 2.53.5\n" + artifacthub.io/changes: "- kind: changed\n description: Bump DefectDojo to 2.54.0\n" diff --git a/helm/defectdojo/README.md b/helm/defectdojo/README.md index be21c4353a6..ca10a9462d8 100644 --- a/helm/defectdojo/README.md +++ b/helm/defectdojo/README.md @@ -511,7 +511,7 @@ The HELM schema will be generated for you. # General information about chart values -![Version: 1.9.5](https://img.shields.io/badge/Version-1.9.5-informational?style=flat-square) ![AppVersion: 2.53.5](https://img.shields.io/badge/AppVersion-2.53.5-informational?style=flat-square) +![Version: 1.9.6](https://img.shields.io/badge/Version-1.9.6-informational?style=flat-square) ![AppVersion: 2.54.0](https://img.shields.io/badge/AppVersion-2.54.0-informational?style=flat-square) A Helm chart for Kubernetes to install DefectDojo @@ -525,7 +525,7 @@ A Helm chart for Kubernetes to install DefectDojo | Repository | Name | Version | |------------|------|---------| -| oci://registry-1.docker.io/cloudpirates | valkey | 0.10.2 | +| oci://registry-1.docker.io/cloudpirates | valkey | 0.13.0 | | oci://us-docker.pkg.dev/os-public-container-registry/defectdojo | postgresql | 16.7.27 | ## Values @@ -589,13 +589,13 @@ A Helm chart for Kubernetes to install DefectDojo | celery.worker.startupProbe | object | `{}` | Enable startup probe for Celery worker container. | | celery.worker.terminationGracePeriodSeconds | int | `300` | | | celery.worker.tolerations | list | `[]` | | -| cloudsql | object | `{"containerSecurityContext":{},"enable_iam_login":false,"enabled":false,"extraEnv":[],"extraVolumeMounts":[],"image":{"pullPolicy":"IfNotPresent","repository":"gcr.io/cloudsql-docker/gce-proxy","tag":"1.37.10"},"instance":"","resources":{},"use_private_ip":false,"verbose":true}` | Google CloudSQL support in GKE via gce-proxy | +| cloudsql | object | `{"containerSecurityContext":{},"enable_iam_login":false,"enabled":false,"extraEnv":[],"extraVolumeMounts":[],"image":{"pullPolicy":"IfNotPresent","repository":"gcr.io/cloudsql-docker/gce-proxy","tag":"1.37.11"},"instance":"","resources":{},"use_private_ip":false,"verbose":true}` | Google CloudSQL support in GKE via gce-proxy | | cloudsql.containerSecurityContext | object | `{}` | Optional: security context for the CloudSQL proxy container. | | cloudsql.enable_iam_login | bool | `false` | use IAM database authentication | | cloudsql.enabled | bool | `false` | To use CloudSQL in GKE set 'enable: true' | | cloudsql.extraEnv | list | `[]` | Additional environment variables for the CloudSQL proxy container. | | cloudsql.extraVolumeMounts | list | `[]` | Array of additional volume mount points for the CloudSQL proxy container | -| cloudsql.image | object | `{"pullPolicy":"IfNotPresent","repository":"gcr.io/cloudsql-docker/gce-proxy","tag":"1.37.10"}` | set repo and image tag of gce-proxy | +| cloudsql.image | object | `{"pullPolicy":"IfNotPresent","repository":"gcr.io/cloudsql-docker/gce-proxy","tag":"1.37.11"}` | set repo and image tag of gce-proxy | | cloudsql.instance | string | `""` | set CloudSQL instance: 'project:zone:instancename' | | cloudsql.resources | object | `{}` | Optional: add resource requests/limits for the CloudSQL proxy container. | | cloudsql.use_private_ip | bool | `false` | whether to use a private IP to connect to the database | diff --git a/helm/defectdojo/values.yaml b/helm/defectdojo/values.yaml index a02a74abcdc..66bdb88efeb 100644 --- a/helm/defectdojo/values.yaml +++ b/helm/defectdojo/values.yaml @@ -601,7 +601,7 @@ cloudsql: # -- set repo and image tag of gce-proxy image: repository: gcr.io/cloudsql-docker/gce-proxy - tag: 1.37.10 + tag: 1.37.11 pullPolicy: IfNotPresent # -- set CloudSQL instance: 'project:zone:instancename' instance: "" diff --git a/requirements-dev.txt b/requirements-dev.txt index 4e8b5cd1fd5..d2a367fca58 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ django-debug-toolbar==6.1.0 django-debug-toolbar-request-history==0.1.4 # Testing dependencies -vcrpy==7.0.0 +vcrpy==8.1.1 vcrpy-unittest==0.1.7 django-test-migrations==1.5.0 parameterized==0.9.0 diff --git a/requirements-lint.txt b/requirements-lint.txt index c7e2cafe88d..76dbc2656d3 100644 --- a/requirements-lint.txt +++ b/requirements-lint.txt @@ -1 +1 @@ -ruff==0.14.6 \ No newline at end of file +ruff==0.14.10 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ec38a520fbf..9691129a8b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,70 +1,71 @@ # requirements.txt for DefectDojo using Python 3.x -asteval==1.0.7 +asteval==1.0.8 bleach==6.3.0 bleach[css] -celery==5.5.3 +celery==5.6.1 defusedxml==0.7.1 django_celery_results==2.6.0 django-auditlog==3.2.1 -django-pghistory==3.8.3 -django-dbbackup==5.0.1 +django-pghistory==3.9.1 +django-dbbackup==5.1.0 django-environ==0.12.0 -django-filter==25.1 +django-filter==25.2 django-imagekit==6.0.0 django-multiselectfield==1.0.1 -django-polymorphic==4.1.0 +django-polymorphic==4.5.2 django-crispy-forms==2.5 django_extensions==4.1 django-slack==5.19.0 django-watson==1.6.3 django-prometheus==2.4.1 -Django==5.1.15 +Django==5.2.9 django-single-session==0.2.0 djangorestframework==3.16.1 html2text==2025.4.15 -humanize==4.14.0 +humanize==4.15.0 jira==3.10.5 PyGithub==2.8.1 lxml==6.0.2 Markdown==3.10 openpyxl==3.1.5 -Pillow==12.0.0 # required by django-imagekit -psycopg[c]==3.2.13 +Pillow==12.1.0 # required by django-imagekit +psycopg[c]==3.3.2 cryptography==46.0.3 python-dateutil==2.9.0.post0 redis==7.1.0 requests==2.32.5 -sqlalchemy==2.0.44 # Required by Celery broker transport -urllib3==2.6.0 +sqlalchemy==2.0.45 # Required by Celery broker transport +urllib3==2.6.2 uWSGI==2.0.31 vobject==0.9.9 whitenoise==5.2.0 titlecase==2.4.1 social-auth-app-django==5.6.0 -social-auth-core==4.8.1 -gitpython==3.1.45 -python-gitlab==7.0.0 +social-auth-core==4.8.3 +gitpython==3.1.46 +python-gitlab==7.1.0 cpe==1.3.1 packageurl-python==0.17.6 django-crum==0.7.9 JSON-log-formatter==1.1.1 django-split-settings==1.3.2 # do not upgrade to 2.1.1 - https://github.com/DefectDojo/django-DefectDojo/issues/12918 -django-tagulous==2.1.0 +# use fork with django 5.2 fixes, but based on 2.1.0 +git+https://github.com/valentijnscholten/django-tagulous.git@2b514f9140acfce608238d1426d864185b3c60a2#egg=django-tagulous PyJWT==2.10.1 cvss==3.6 -django-fieldsignals==0.7.0 +django-fieldsignals==0.8.0 hyperlink==21.0.0 djangosaml2==1.11.1 drf-spectacular==0.29.0 -drf-spectacular-sidecar==2025.10.1 +drf-spectacular-sidecar==2026.1.1 django-ratelimit==4.1.0 argon2-cffi==25.1.0 blackduck==1.1.3 pycurl==7.45.7 # Required for Celery Broker AWS (SQS) support boto3==1.41.5 # Required for Celery Broker AWS (SQS) support netaddr==1.3.0 -vulners==3.1.2 +vulners==3.1.3 fontawesomefree==6.6.0 PyYAML==6.0.3 pyopenssl==25.3.0 diff --git a/ruff.toml b/ruff.toml index 670a95b7b99..942bd47a00a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -84,7 +84,7 @@ select = [ "PGH", "PLC", "PLE", - "PLR01", "PLR02", "PLR04", "PLR0915", "PLR1711", "PLR1704", "PLR1714", "PLR1716", "PLR172", "PLR173", "PLR2044", "PLR5", "PLR6104", "PLR6201", + "PLR01", "PLR02", "PLR04", "PLR0915", "PLR1711", "PLR1704", "PLR1714", "PLR1716", "PLR172", "PLR173", "PLR2044", "PLR5", "PLR6104", "PLR6201", "PLW", "UP", "FURB", @@ -118,6 +118,15 @@ preview = true "dojo/filters.py" = [ "A003", # ruff upgrade to 0.13.3 ] +"scripts/update_performance_test_counts.py" = [ + "S603", # subprocess.run without shell=True is safe for this script + "S604", # subprocess.run without shell=True is safe for this script + "S605", # subprocess.run without shell=True is safe for this script + "S606", # subprocess.run without shell=True is safe for this script + "S607", # subprocess.run without shell=True is safe for this script + "T201", # print statements are fine for console output scripts + "EXE001", # git won't commit the executable flag I don't know why +] [lint.flake8-boolean-trap] extend-allowed-calls = ["dojo.utils.get_system_setting"] diff --git a/scripts/update_performance_test_counts.py b/scripts/update_performance_test_counts.py new file mode 100644 index 00000000000..f7cfaae2859 --- /dev/null +++ b/scripts/update_performance_test_counts.py @@ -0,0 +1,667 @@ +#!/usr/bin/env python3 +""" +Script to update performance test query counts. +I tried to implement this as management command, but it would become very slow on parsing the test output. + +This script: +1. Runs all performance tests and captures actual query counts +2. Compares them with expected counts +3. Generates a report and optionally updates the test file +4. Provides a verification run + +How to run: + + # Default: Update the test file (uses TestDojoImporterPerformanceSmall by default) + python3 scripts/update_performance_test_counts.py + + # Or specify a different test class: + python3 scripts/update_performance_test_counts.py --test-class TestDojoImporterPerformanceSmall + + # Step 1: Run tests and generate report only (without updating) + python3 scripts/update_performance_test_counts.py --report-only + + # Step 2: Verify all tests pass + python3 scripts/update_performance_test_counts.py --verify + +The script defaults to TestDojoImporterPerformanceSmall if --test-class is not provided. +The script defaults to --update behavior if no action flag is provided. +""" + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +# Path to the test file +TEST_FILE = Path(__file__).parent.parent / "unittests" / "test_importers_performance.py" + + +class TestCount: + + """Represents a test's expected and actual counts.""" + + def __init__(self, test_name: str, step: str, metric: str): + self.test_name = test_name + self.step = step + self.metric = metric + self.expected = None + self.actual = None + self.difference = None + + def __repr__(self): + return ( + f"TestCount({self.test_name}, {self.step}, {self.metric}, " + f"expected={self.expected}, actual={self.actual})" + ) + + +def extract_test_methods(test_class: str) -> list[str]: + """Extract all test method names from the test class.""" + if not TEST_FILE.exists(): + msg = f"Test file not found: {TEST_FILE}" + raise FileNotFoundError(msg) + + content = TEST_FILE.read_text() + + # Find the test class definition + class_pattern = re.compile( + rf"class {re.escape(test_class)}.*?(?=class |\Z)", + re.DOTALL, + ) + class_match = class_pattern.search(content) + if not class_match: + return [] + + class_content = class_match.group(0) + + # Find all test methods in this class + test_method_pattern = re.compile(r"def (test_\w+)\(") + return test_method_pattern.findall(class_content) + + +def run_test_method(test_class: str, test_method: str) -> tuple[str, int]: + """Run a specific test method and return the output and return code.""" + print(f"Running {test_class}.{test_method}...") + cmd = [ + "./run-unittest.sh", + "--test-case", + f"unittests.test_importers_performance.{test_class}.{test_method}", + ] + + # Run with real-time output streaming + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + cwd=Path(__file__).parent.parent, + ) + + output_lines = [] + for line in process.stdout: + print(line, end="") # Print in real-time + output_lines.append(line) + + process.wait() + output = "".join(output_lines) + + return output, process.returncode + + +def run_tests(test_class: str) -> tuple[str, int]: + """Run all tests in a test class and return the output and return code.""" + print(f"Running tests for {test_class}...") + cmd = [ + "./run-unittest.sh", + "--test-case", + f"unittests.test_importers_performance.{test_class}", + ] + + # Run with real-time output streaming + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + cwd=Path(__file__).parent.parent, + ) + + output_lines = [] + for line in process.stdout: + print(line, end="") # Print in real-time + output_lines.append(line) + + process.wait() + output = "".join(output_lines) + + return output, process.returncode + + +def check_test_execution_success(output: str, return_code: int) -> tuple[bool, str]: + """Check if tests executed successfully or failed due to other reasons.""" + # Check for migration errors + migration_error_patterns = [ + r"django\.db\.migrations\.exceptions\.", + r"Migration.*failed", + r"django\.core\.management\.base\.CommandError", + r"OperationalError", + r"ProgrammingError", + r"relation.*does not exist", + r"no such table", + ] + + for pattern in migration_error_patterns: + if re.search(pattern, output, re.IGNORECASE): + return False, f"Migration or database error detected: {pattern}" + + # Check if any tests actually ran + test_run_patterns = [ + r"Ran \d+ test", + r"OK", + r"FAILED", + r"FAIL:", + r"test_\w+", + ] + + tests_ran = any(re.search(pattern, output) for pattern in test_run_patterns) + + if not tests_ran and return_code != 0: + return False, "Tests did not run successfully. Check the output above for errors." + + # Check for other critical errors + critical_error_patterns = [ + r"ImportError", + r"ModuleNotFoundError", + r"SyntaxError", + r"IndentationError", + ] + + for pattern in critical_error_patterns: + if re.search(pattern, output): + return False, f"Critical error detected: {pattern}" + + return True, "" + + +def parse_test_output(output: str) -> list[TestCount]: + """Parse test output to extract actual vs expected counts.""" + counts = [] + + # Debug: Save a sample of the output to help diagnose parsing issues + if "FAIL:" in output: + # Extract failure sections for debugging + fail_sections = [] + lines = output.split("\n") + in_fail_section = False + fail_section = [] + for line in lines: + if "FAIL:" in line: + if fail_section: + fail_sections.append("\n".join(fail_section)) + fail_section = [line] + in_fail_section = True + elif in_fail_section: + fail_section.append(line) + # Stop collecting after AssertionError line or after 5 more lines + if "AssertionError:" in line or len(fail_section) > 6: + fail_sections.append("\n".join(fail_section)) + fail_section = [] + in_fail_section = False + if fail_section: + fail_sections.append("\n".join(fail_section)) + + if fail_sections: + print(f"\n🔍 Found {len(fail_sections)} failure section(s) in output") + + # The test output format is: + # FAIL: test_name (step='import1', metric='queries') + # AssertionError: 118 != 120 : 118 queries executed, 120 expected + # OR for async tasks: + # FAIL: test_name (step='import1', metric='async_tasks') + # AssertionError: 7 != 8 : 7 async tasks executed, 8 expected + + # Pattern to match the full failure block: + # FAIL: test_name (full.path.to.test) (step='...', metric='...') + # AssertionError: actual != expected : actual ... executed, expected expected + # The test name may include the full path in parentheses, so we extract just the method name + failure_pattern = re.compile( + r"FAIL:\s+(test_\w+)\s+\([^)]+\)\s+\(step=['\"](\w+)['\"],\s*metric=['\"](\w+)['\"]\)\s*\n" + r".*?AssertionError:\s+(\d+)\s+!=\s+(\d+)\s+:\s+\d+\s+(?:queries|async tasks?)\s+executed,\s+\d+\s+expected", + re.MULTILINE | re.DOTALL, + ) + + for match in failure_pattern.finditer(output): + test_name = match.group(1) + step = match.group(2) + metric = match.group(3) + actual = int(match.group(4)) + expected = int(match.group(5)) + + count = TestCount(test_name, step, metric) + count.actual = actual + count.expected = expected + count.difference = expected - actual + counts.append(count) + + # Also try a simpler pattern in case the format is slightly different + if not counts: + # Look for lines with step/metric followed by AssertionError on nearby lines + lines = output.split("\n") + i = 0 + while i < len(lines): + line = lines[i] + + # Look for FAIL: test_name (may include full path in parentheses) + # Format: FAIL: test_name (full.path) (step='...', metric='...') + fail_match = re.search(r"FAIL:\s+(test_\w+)\s+\([^)]+\)\s+\(step=['\"](\w+)['\"],\s*metric=['\"](\w+)['\"]\)", line) + if fail_match: + test_name = fail_match.group(1) + step = fail_match.group(2) + metric = fail_match.group(3) + # Look ahead for AssertionError + for j in range(i, min(i + 15, len(lines))): + assertion_match = re.search( + r"AssertionError:\s+(\d+)\s+!=\s+(\d+)\s+:\s+\d+\s+(?:queries|async tasks?)\s+executed,\s+\d+\s+expected", + lines[j], + ) + + if assertion_match: + actual = int(assertion_match.group(1)) + expected = int(assertion_match.group(2)) + + count = TestCount(test_name, step, metric) + count.actual = actual + count.expected = expected + count.difference = expected - actual + counts.append(count) + break + i += 1 + + if counts: + print(f"\n📊 Parsed {len(counts)} count mismatch(es) from test output:") + for count in counts: + print(f" {count.test_name} - {count.step} {count.metric}: {count.actual} != {count.expected}") + elif "FAIL:" in output: + print("\n⚠️ WARNING: Found FAIL in output but couldn't parse any count mismatches!") + print("This might indicate a parsing issue. Check the output above.") + + return counts + + +def extract_expected_counts_from_file(test_class: str) -> dict[str, dict[str, int]]: + """Extract expected counts from the test file.""" + if not TEST_FILE.exists(): + msg = f"Test file not found: {TEST_FILE}" + raise FileNotFoundError(msg) + + content = TEST_FILE.read_text() + + # Pattern to match test method calls with expected counts + # Format: self._import_reimport_performance( + # expected_num_queries1=340, + # expected_num_async_tasks1=7, + # expected_num_queries2=238, + # expected_num_async_tasks2=18, + # expected_num_queries3=120, + # expected_num_async_tasks3=17, + # ) + # More flexible pattern that handles whitespace variations + pattern = re.compile( + r"def (test_\w+)\([^)]*\):.*?" + r"self\._import_reimport_performance\(\s*" + r"expected_num_queries1\s*=\s*(\d+)\s*,\s*" + r"expected_num_async_tasks1\s*=\s*(\d+)\s*,\s*" + r"expected_num_queries2\s*=\s*(\d+)\s*,\s*" + r"expected_num_async_tasks2\s*=\s*(\d+)\s*,\s*" + r"expected_num_queries3\s*=\s*(\d+)\s*,\s*" + r"expected_num_async_tasks3\s*=\s*(\d+)\s*,", + re.DOTALL, + ) + + expected_counts = {} + for match in pattern.finditer(content): + test_name = match.group(1) + expected_counts[test_name] = { + "import1_queries": int(match.group(2)), + "import1_async_tasks": int(match.group(3)), + "reimport1_queries": int(match.group(4)), + "reimport1_async_tasks": int(match.group(5)), + "reimport2_queries": int(match.group(6)), + "reimport2_async_tasks": int(match.group(7)), + } + + return expected_counts + + +def generate_report(counts: list[TestCount], expected_counts: dict[str, dict[str, int]]): + """Generate a report of differences.""" + if not counts: + print("✅ All tests passed! No count differences found.") + return + + print("\n" + "=" * 80) + print("PERFORMANCE TEST COUNT DIFFERENCES REPORT") + print("=" * 80 + "\n") + + # Group by test name + by_test = {} + for count in counts: + if count.test_name not in by_test: + by_test[count.test_name] = [] + by_test[count.test_name].append(count) + + for test_name, test_counts in sorted(by_test.items()): + print(f"Test: {test_name}") + print("-" * 80) + for count in sorted(test_counts, key=lambda x: (x.step, x.metric)): + print( + f" {count.step:12} {count.metric:15} " + f"Expected: {count.expected:4} → Actual: {count.actual:4} " + f"(Difference: {count.difference:+3})", + ) + print() + + print("=" * 80) + print("\nTo update the test file, run:") + print(f" python scripts/update_performance_test_counts.py --test-class {test_name.split('_')[0]} --update") + print() + + +def update_test_file(counts: list[TestCount]): + """Update the test file with new expected counts.""" + if not counts: + print("No counts to update.") + return + + content = TEST_FILE.read_text() + + # Create a mapping of test_name -> step_metric -> new_value + updates = {} + for count in counts: + if count.test_name not in updates: + updates[count.test_name] = {} + step_metric = f"{count.step}_{count.metric}" + updates[count.test_name][step_metric] = count.actual + + # Map step_metric to parameter name for different methods + param_map_import_reimport = { + "import1_queries": "expected_num_queries1", + "import1_async_tasks": "expected_num_async_tasks1", + "reimport1_queries": "expected_num_queries2", + "reimport1_async_tasks": "expected_num_async_tasks2", + "reimport2_queries": "expected_num_queries3", + "reimport2_async_tasks": "expected_num_async_tasks3", + } + param_map_deduplication = { + "first_import_queries": "expected_num_queries1", + "first_import_async_tasks": "expected_num_async_tasks1", + "second_import_queries": "expected_num_queries2", + "second_import_async_tasks": "expected_num_async_tasks2", + } + + # Update each test method + for test_name, test_updates in updates.items(): + print(f" Updating {test_name}...") + # Find the test method boundaries + test_method_pattern = re.compile( + rf"(def {re.escape(test_name)}\([^)]*\):.*?)(?=def test_|\Z)", + re.DOTALL, + ) + test_match = test_method_pattern.search(content) + if not test_match: + print(f"⚠️ Warning: Could not find test method {test_name}") + continue + + test_method_content = test_match.group(1) + test_method_start = test_match.start() + test_method_end = test_match.end() + + # Try to find _import_reimport_performance call first + perf_call_pattern_import_reimport = re.compile( + r"(self\._import_reimport_performance\s*\(\s*)" + r"expected_num_queries1\s*=\s*(\d+)\s*,\s*" + r"expected_num_async_tasks1\s*=\s*(\d+)\s*,\s*" + r"expected_num_queries2\s*=\s*(\d+)\s*,\s*" + r"expected_num_async_tasks2\s*=\s*(\d+)\s*,\s*" + r"expected_num_queries3\s*=\s*(\d+)\s*,\s*" + r"expected_num_async_tasks3\s*=\s*(\d+)\s*," + r"(\s*\))", + re.DOTALL, + ) + + # Try to find _deduplication_performance call + perf_call_pattern_deduplication = re.compile( + r"(self\._deduplication_performance\s*\(\s*)" + r"expected_num_queries1\s*=\s*(\d+)\s*,\s*" + r"expected_num_async_tasks1\s*=\s*(\d+)\s*,\s*" + r"expected_num_queries2\s*=\s*(\d+)\s*,\s*" + r"expected_num_async_tasks2\s*=\s*(\d+)\s*," + r"(\s*\))", + re.DOTALL, + ) + + perf_match = perf_call_pattern_import_reimport.search(test_method_content) + method_type = "import_reimport" + param_map = param_map_import_reimport + param_order = [ + "import1_queries", + "import1_async_tasks", + "reimport1_queries", + "reimport1_async_tasks", + "reimport2_queries", + "reimport2_async_tasks", + ] + + if not perf_match: + perf_match = perf_call_pattern_deduplication.search(test_method_content) + if perf_match: + method_type = "deduplication" + param_map = param_map_deduplication + param_order = [ + "first_import_queries", + "first_import_async_tasks", + "second_import_queries", + "second_import_async_tasks", + ] + else: + print(f"⚠️ Warning: Could not find _import_reimport_performance or _deduplication_performance call in {test_name}") + continue + + # Get the indentation from the original call (first line after opening paren) + call_lines = test_method_content[perf_match.start():perf_match.end()].split("\n") + indent = "" + for line in call_lines: + if "expected_num_queries1" in line: + # Extract indentation (spaces before the parameter) + indent_match = re.match(r"(\s*)expected_num_queries1", line) + if indent_match: + indent = indent_match.group(1) + break + + # If we couldn't find indentation, use a default + if not indent: + indent = " " # 12 spaces default + + replacement_parts = [perf_match.group(1)] # Opening: "self._import_reimport_performance(" + updated_params = [] + for i, step_metric in enumerate(param_order): + param_name = param_map[step_metric] + old_value = int(perf_match.group(i + 2)) # +2 because group 1 is the opening + if step_metric in test_updates: + new_value = test_updates[step_metric] + if old_value != new_value: + updated_params.append(f"{param_name}: {old_value} → {new_value}") + else: + # Keep the existing value + new_value = old_value + + replacement_parts.append(f"{indent}{param_name}={new_value},") + + # Closing parenthesis - group number depends on method type + closing_group = 8 if method_type == "import_reimport" else 6 + replacement_parts.append(perf_match.group(closing_group)) # Closing parenthesis + replacement = "\n".join(replacement_parts) + + if updated_params: + print(f" Updated: {', '.join(updated_params)}") + + # Replace the method call within the test method content + updated_method_content = ( + test_method_content[: perf_match.start()] + + replacement + + test_method_content[perf_match.end() :] + ) + + # Replace the entire test method in the original content + content = content[:test_method_start] + updated_method_content + content[test_method_end:] + + # Write back to file + TEST_FILE.write_text(content) + print(f"✅ Updated {TEST_FILE}") + print(f" Updated {len(counts)} count(s) across {len(updates)} test(s)") + + +def verify_tests(test_class: str) -> bool: + """Run tests to verify they all pass.""" + print(f"Verifying tests for {test_class}...") + output, return_code = run_tests(test_class) + + success, error_msg = check_test_execution_success(output, return_code) + if not success: + print(f"\n❌ Test execution failed: {error_msg}") + return False + + counts = parse_test_output(output) + + if counts: + print("\n❌ Some tests still have count mismatches:") + for count in counts: + print(f" {count.test_name} - {count.step} {count.metric}: " + f"expected {count.expected}, got {count.actual}") + return False + else: # noqa: RET505 + print("\n✅ All tests pass!") + return True + + +def main(): + parser = argparse.ArgumentParser( + description="Update performance test query counts", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "--test-class", + required=False, + default="TestDojoImporterPerformanceSmall", + help="Test class name (e.g., TestDojoImporterPerformanceSmall). Defaults to TestDojoImporterPerformanceSmall if not provided.", + ) + parser.add_argument( + "--report-only", + action="store_true", + help="Only generate a report, don't update the file", + ) + parser.add_argument( + "--update", + action="store_true", + help="Update the test file with new counts (default behavior if no action flag is provided)", + ) + parser.add_argument( + "--verify", + action="store_true", + help="Run tests to verify they pass", + ) + + args = parser.parse_args() + + if args.report_only: + # Step 1: Run tests and generate report + # Run each test method individually + test_methods = extract_test_methods(args.test_class) + if not test_methods: + print(f"⚠️ No test methods found in {args.test_class}") + sys.exit(1) + + print(f"\nFound {len(test_methods)} test method(s) in {args.test_class}") + print("=" * 80) + + all_counts = [] + for test_method in test_methods: + print(f"\n{'=' * 80}") + output, return_code = run_test_method(args.test_class, test_method) + success, error_msg = check_test_execution_success(output, return_code) + if not success: + print(f"\n⚠️ Test execution failed for {test_method}: {error_msg}") + print("Skipping this test method...") + continue + + counts = parse_test_output(output) + if counts: + all_counts.extend(counts) + + expected_counts = extract_expected_counts_from_file(args.test_class) + generate_report(all_counts, expected_counts) + + elif args.verify: + # Step 3: Verify + success = verify_tests(args.test_class) + sys.exit(0 if success else 1) + + else: + # Default: Update the file (--update is the default behavior) + # Run each test method individually + test_methods = extract_test_methods(args.test_class) + if not test_methods: + print(f"⚠️ No test methods found in {args.test_class}") + sys.exit(1) + + print(f"\nFound {len(test_methods)} test method(s) in {args.test_class}") + print("=" * 80) + + all_counts = [] + for test_method in test_methods: + print(f"\n{'=' * 80}") + output, return_code = run_test_method(args.test_class, test_method) + success, error_msg = check_test_execution_success(output, return_code) + if not success: + print(f"\n⚠️ Test execution failed for {test_method}: {error_msg}") + print("Skipping this test method...") + continue + + counts = parse_test_output(output) + + # Check if test actually passed + test_passed = "OK" in output or ("Ran" in output and "FAILED" not in output and return_code == 0) + + if counts: + all_counts.extend(counts) + # Update immediately after each test + update_test_file(counts) + print(f"⚠️ {test_method}: Found {len(counts)} count mismatch(es) - updated file") + elif test_passed: + print(f"✅ {test_method}: Test passed, all counts match") + elif return_code != 0: + # Test might have failed for other reasons + print(f"⚠️ {test_method}: Test failed (exit code {return_code}) but no count mismatches parsed") + print(" This might indicate a parsing issue or a different type of failure") + # Show a snippet of the output to help debug + fail_lines = [line for line in output.split("\n") if "FAIL" in line or "Error" in line or "Exception" in line] + if fail_lines: + print(" Relevant error lines:") + for line in fail_lines[:5]: + print(f" {line}") + + if all_counts: + print(f"\n{'=' * 80}") + print(f"✅ Updated {len(all_counts)} count(s) across {len({c.test_name for c in all_counts})} test(s)") + print("\nNext step: Run --verify to ensure all tests pass") + else: + print(f"\n{'=' * 80}") + print("\n✅ No differences found. All tests are already up to date.") + + +if __name__ == "__main__": + main() diff --git a/unittests/scans/anchore_grype/check_all_fields_different_ids_fabricated.json b/unittests/scans/anchore_grype/check_all_fields_different_ids_fabricated.json new file mode 100644 index 00000000000..8520e8828dc --- /dev/null +++ b/unittests/scans/anchore_grype/check_all_fields_different_ids_fabricated.json @@ -0,0 +1,167 @@ +{ + "matches": [ + { + "vulnerability": { + "id": "GHSA-v6rh-hp5x-86rv", + "dataSource": "https://github.com/advisories/GHSA-v6rh-hp5x-86rv", + "namespace": "github:python", + "severity": "High", + "urls": [ + "https://github.com/advisories/GHSA-v6rh-hp5x-86rv" + ], + "description": "Potential bypass of an upstream access control based on URL paths in Django", + "cvss": [], + "fix": { + "versions": [ + "3.2.10" + ], + "state": "fixed" + }, + "advisories": [] + }, + "relatedVulnerabilities": [ + { + "id": "CVE-2021-1234", + "dataSource": "https://nvd.nist.gov/vuln/detail/CVE-2021-1234", + "namespace": "nvd", + "severity": "High", + "urls": [ + "https://example.com/cve-2021-1234" + ], + "description": "A different CVE for testing vulnerability ID changes", + "cvss": [ + { + "version": "3.1", + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "metrics": { + "baseScore": 9.8, + "exploitabilityScore": 3.9, + "impactScore": 5.9 + }, + "vendorMetadata": {} + } + ] + }, + { + "id": "CVE-2021-5678", + "dataSource": "https://nvd.nist.gov/vuln/detail/CVE-2021-5678", + "namespace": "nvd", + "severity": "Medium", + "urls": [ + "https://example.com/cve-2021-5678" + ], + "description": "Another different CVE for testing vulnerability ID changes", + "cvss": [ + { + "version": "3.1", + "vector": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:L/I:L/A:L", + "metrics": { + "baseScore": 6.3, + "exploitabilityScore": 3.9, + "impactScore": 2.4 + }, + "vendorMetadata": {} + } + ] + } + ], + "matchDetails": [ + { + "matcher": "python-matcher", + "searchedBy": { + "language": "python", + "namespace": "github:python" + }, + "found": { + "versionConstraint": ">=3.2,<3.2.10 (python)" + } + } + ], + "artifact": { + "name": "Django", + "version": "3.2.9", + "type": "python", + "locations": [ + { + "path": "/usr/local/lib/python3.8/site-packages/Django-3.2.9.dist-info/METADATA", + "layerID": "sha256:b1d4455cf82b15a50b006fe87bd29f694c8f9155456253eb67fdd155b5edcf4a" + } + ], + "language": "python", + "licenses": [ + "BSD-3-Clause" + ], + "cpes": [ + "cpe:2.3:a:django_software_foundation:Django:3.2.9:*:*:*:*:*:*:*" + ], + "purl": "pkg:pypi/Django@3.2.9", + "metadata": null + } + } + ], + "source": { + "type": "image", + "target": { + "userInput": "vulnerable-image:latest", + "imageID": "sha256:ce9898fd214aef9c994a42624b09056bdce3ff4a8e3f68dc242d967b80fcbeee", + "manifestDigest": "sha256:9d8825ab20ac86b40eb71495bece1608a302fb180384740697a28c2b0a5a0fc6", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "tags": [ + "vulnerable-image:latest" + ], + "imageSize": 707381791, + "layers": [] + } + }, + "distro": { + "name": "debian", + "version": "10", + "idLike": "" + }, + "descriptor": { + "name": "grype", + "version": "0.28.0", + "configuration": { + "configPath": "", + "output": "json", + "file": "", + "output-template-file": "", + "quiet": false, + "check-for-app-update": true, + "only-fixed": false, + "scope": "Squashed", + "log": { + "structured": false, + "level": "", + "file": "" + }, + "db": { + "cache-dir": "/home/user/.cache/grype/db", + "update-url": "https://toolbox-data.anchore.io/grype/databases/listing.json", + "ca-cert": "", + "auto-update": true, + "validate-by-hash-on-start": false + }, + "dev": { + "profile-cpu": false, + "profile-mem": false + }, + "fail-on-severity": "", + "registry": { + "insecure-skip-tls-verify": false, + "insecure-use-http": false, + "auth": [] + }, + "ignore": null, + "exclude": [] + }, + "db": { + "built": "2021-12-24T08:14:02Z", + "schemaVersion": 3, + "location": "/home/user/.cache/grype/db/3", + "checksum": "sha256:6c4777e1acea787e5335ccee6b5e4562cd1767b9cca138c07e0802efb2a74162", + "error": null + } + } + } + diff --git a/unittests/scans/anchore_grype/check_all_fields_no_ids_fabricated.json b/unittests/scans/anchore_grype/check_all_fields_no_ids_fabricated.json new file mode 100644 index 00000000000..d4371d3a478 --- /dev/null +++ b/unittests/scans/anchore_grype/check_all_fields_no_ids_fabricated.json @@ -0,0 +1,122 @@ +{ + "matches": [ + { + "vulnerability": { + "id": "GHSA-v6rh-hp5x-86rv", + "dataSource": "https://github.com/advisories/GHSA-v6rh-hp5x-86rv", + "namespace": "github:python", + "severity": "High", + "urls": [ + "https://github.com/advisories/GHSA-v6rh-hp5x-86rv" + ], + "description": "Potential bypass of an upstream access control based on URL paths in Django", + "cvss": [], + "fix": { + "versions": [ + "3.2.10" + ], + "state": "fixed" + }, + "advisories": [] + }, + "relatedVulnerabilities": [], + "matchDetails": [ + { + "matcher": "python-matcher", + "searchedBy": { + "language": "python", + "namespace": "github:python" + }, + "found": { + "versionConstraint": ">=3.2,<3.2.10 (python)" + } + } + ], + "artifact": { + "name": "Django", + "version": "3.2.9", + "type": "python", + "locations": [ + { + "path": "/usr/local/lib/python3.8/site-packages/Django-3.2.9.dist-info/METADATA", + "layerID": "sha256:b1d4455cf82b15a50b006fe87bd29f694c8f9155456253eb67fdd155b5edcf4a" + } + ], + "language": "python", + "licenses": [ + "BSD-3-Clause" + ], + "cpes": [ + "cpe:2.3:a:django_software_foundation:Django:3.2.9:*:*:*:*:*:*:*" + ], + "purl": "pkg:pypi/Django@3.2.9", + "metadata": null + } + } + ], + "source": { + "type": "image", + "target": { + "userInput": "vulnerable-image:latest", + "imageID": "sha256:ce9898fd214aef9c994a42624b09056bdce3ff4a8e3f68dc242d967b80fcbeee", + "manifestDigest": "sha256:9d8825ab20ac86b40eb71495bece1608a302fb180384740697a28c2b0a5a0fc6", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "tags": [ + "vulnerable-image:latest" + ], + "imageSize": 707381791, + "layers": [] + } + }, + "distro": { + "name": "debian", + "version": "10", + "idLike": "" + }, + "descriptor": { + "name": "grype", + "version": "0.28.0", + "configuration": { + "configPath": "", + "output": "json", + "file": "", + "output-template-file": "", + "quiet": false, + "check-for-app-update": true, + "only-fixed": false, + "scope": "Squashed", + "log": { + "structured": false, + "level": "", + "file": "" + }, + "db": { + "cache-dir": "/home/user/.cache/grype/db", + "update-url": "https://toolbox-data.anchore.io/grype/databases/listing.json", + "ca-cert": "", + "auto-update": true, + "validate-by-hash-on-start": false + }, + "dev": { + "profile-cpu": false, + "profile-mem": false + }, + "fail-on-severity": "", + "registry": { + "insecure-skip-tls-verify": false, + "insecure-use-http": false, + "auth": [] + }, + "ignore": null, + "exclude": [] + }, + "db": { + "built": "2021-12-24T08:14:02Z", + "schemaVersion": 3, + "location": "/home/user/.cache/grype/db/3", + "checksum": "sha256:6c4777e1acea787e5335ccee6b5e4562cd1767b9cca138c07e0802efb2a74162", + "error": null + } + } + } + diff --git a/unittests/scans/anchore_grype/check_all_fields_with_ids_fabricated.json b/unittests/scans/anchore_grype/check_all_fields_with_ids_fabricated.json new file mode 100644 index 00000000000..028bd2c7b7b --- /dev/null +++ b/unittests/scans/anchore_grype/check_all_fields_with_ids_fabricated.json @@ -0,0 +1,158 @@ +{ + "matches": [ + { + "vulnerability": { + "id": "GHSA-v6rh-hp5x-86rv", + "dataSource": "https://github.com/advisories/GHSA-v6rh-hp5x-86rv", + "namespace": "github:python", + "severity": "High", + "urls": [ + "https://github.com/advisories/GHSA-v6rh-hp5x-86rv" + ], + "description": "Potential bypass of an upstream access control based on URL paths in Django", + "cvss": [], + "fix": { + "versions": [ + "3.2.10" + ], + "state": "fixed" + }, + "advisories": [] + }, + "relatedVulnerabilities": [ + { + "id": "CVE-2021-44420", + "dataSource": "https://nvd.nist.gov/vuln/detail/CVE-2021-44420", + "namespace": "nvd", + "severity": "High", + "urls": [ + "https://docs.djangoproject.com/en/3.2/releases/security/", + "https://www.openwall.com/lists/oss-security/2021/12/07/1", + "https://www.djangoproject.com/weblog/2021/dec/07/security-releases/", + "https://groups.google.com/forum/#!forum/django-announce" + ], + "description": "In Django 2.2 before 2.2.25, 3.1 before 3.1.14, and 3.2 before 3.2.10, HTTP requests for URLs with trailing newlines could bypass upstream access control based on URL paths.", + "cvss": [ + { + "version": "2.0", + "vector": "AV:N/AC:L/Au:N/C:P/I:P/A:P", + "metrics": { + "baseScore": 7.5, + "exploitabilityScore": 10, + "impactScore": 6.4 + }, + "vendorMetadata": {} + }, + { + "version": "3.1", + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L", + "metrics": { + "baseScore": 7.3, + "exploitabilityScore": 3.9, + "impactScore": 3.4 + }, + "vendorMetadata": {} + } + ] + } + ], + "matchDetails": [ + { + "matcher": "python-matcher", + "searchedBy": { + "language": "python", + "namespace": "github:python" + }, + "found": { + "versionConstraint": ">=3.2,<3.2.10 (python)" + } + } + ], + "artifact": { + "name": "Django", + "version": "3.2.9", + "type": "python", + "locations": [ + { + "path": "/usr/local/lib/python3.8/site-packages/Django-3.2.9.dist-info/METADATA", + "layerID": "sha256:b1d4455cf82b15a50b006fe87bd29f694c8f9155456253eb67fdd155b5edcf4a" + } + ], + "language": "python", + "licenses": [ + "BSD-3-Clause" + ], + "cpes": [ + "cpe:2.3:a:django_software_foundation:Django:3.2.9:*:*:*:*:*:*:*" + ], + "purl": "pkg:pypi/Django@3.2.9", + "metadata": null + } + } + ], + "source": { + "type": "image", + "target": { + "userInput": "vulnerable-image:latest", + "imageID": "sha256:ce9898fd214aef9c994a42624b09056bdce3ff4a8e3f68dc242d967b80fcbeee", + "manifestDigest": "sha256:9d8825ab20ac86b40eb71495bece1608a302fb180384740697a28c2b0a5a0fc6", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "tags": [ + "vulnerable-image:latest" + ], + "imageSize": 707381791, + "layers": [] + } + }, + "distro": { + "name": "debian", + "version": "10", + "idLike": "" + }, + "descriptor": { + "name": "grype", + "version": "0.28.0", + "configuration": { + "configPath": "", + "output": "json", + "file": "", + "output-template-file": "", + "quiet": false, + "check-for-app-update": true, + "only-fixed": false, + "scope": "Squashed", + "log": { + "structured": false, + "level": "", + "file": "" + }, + "db": { + "cache-dir": "/home/user/.cache/grype/db", + "update-url": "https://toolbox-data.anchore.io/grype/databases/listing.json", + "ca-cert": "", + "auto-update": true, + "validate-by-hash-on-start": false + }, + "dev": { + "profile-cpu": false, + "profile-mem": false + }, + "fail-on-severity": "", + "registry": { + "insecure-skip-tls-verify": false, + "insecure-use-http": false, + "auth": [] + }, + "ignore": null, + "exclude": [] + }, + "db": { + "built": "2021-12-24T08:14:02Z", + "schemaVersion": 3, + "location": "/home/user/.cache/grype/db/3", + "checksum": "sha256:6c4777e1acea787e5335ccee6b5e4562cd1767b9cca138c07e0802efb2a74162", + "error": null + } + } + } + diff --git a/unittests/scans/pingcastle/many.xml b/unittests/scans/pingcastle/many.xml new file mode 100644 index 00000000000..33080ff4ac0 --- /dev/null +++ b/unittests/scans/pingcastle/many.xml @@ -0,0 +1,927 @@ + + + 3.2.0.1 + 2024-06-06T13:01:09.2643869+02:00 + Normal + 2 + example.local + EXAMPLE + example.local + 2024-03-13T00:24:53 + S-1-5-21-1111111111-2222222222-3333333333 + 7 + 7 + 88 + 0 + false + 2024-03-13T00:25:36 + 0001-01-01T00:00:00 + 1 + 70 + 36 + 70 + 0 + 65 + 0001-01-01T00:00:00 + 0 + + + + DC-NQ7M31 + 2024-03-13T00:25:35 + 2024-06-06T12:14:42.680185+02:00 + 2024-06-06T09:05:02.4862126+02:00 + CN=DC-NQ7M31,OU=Domain Controllers,DC=example,DC=local + Server OS + S-1-5-21-1111111111-2222222222-3333333333-512 + EXAMPLE\Domain Admins + false + false + NotTested + true + SmbSigningEnabled SmbSigningRequired + true + + fe80::2a9 + 198.51.10 + + + PDC + RID pool manager + Infrastructure master + Schema master + Domain naming Master + + + false + false + 2024-06-06T09:04:51.018471+02:00 + false + false + + + + + + + + + + + + + + Default-First-Site-Name + + + + + false + false + true + false + + + 20 + PrivilegedAccounts + AdminControl + P-AdminLogin + The native administrator account has been used recently: 0 day(s) ago + + + 20 + PrivilegedAccounts + AccountTakeOver + P-Delegated + Presence of Admin accounts which do not have the flag "This account is sensitive and cannot be delegated": 7 + + + 15 + StaleObjects + OldAuthenticationProtocols + S-OldNtlm + The LAN Manager Authentication Level allows the use of NTLMv1 or LM. + + + 15 + Anomalies + PassTheCredential + A-LAPS-Not-Installed + LAPS doesn't seem to be installed + + + 10 + PrivilegedAccounts + IrreversibleChange + P-SchemaAdmin + The group Schema Admins is not empty: 1 account(s) + + + 10 + Anomalies + WeakPassword + A-MinPwdLen + Policy where the password length is less than 8 characters: 1 + + + 10 + StaleObjects + Provisioning + S-ADRegistration + Non-admin users can add up to 10 computer(s) to a domain + + + 10 + Anomalies + PassTheCredential + A-DC-Spooler + The spooler service is remotely accessible from 1 DC + + + 10 + PrivilegedAccounts + IrreversibleChange + P-RecycleBin + The Recycle Bin is not enabled + + + 10 + PrivilegedAccounts + AccountTakeOver + P-ProtectedUsers + Number of admins not in Protected Users: 7 + + + 10 + Anomalies + Audit + A-AuditDC + The audit policy on domain controllers does not collect key events. + + + 10 + Anomalies + PassTheCredential + A-DC-Coerce + RPC interfaces of DC are likely vulnerable to coercion attacks. Identified interfaces: 6 + + + 5 + StaleObjects + ObjectConfig + S-NoPreAuth + Number of accounts which do not require Kerberos pre-authentication: 1 + + + 5 + Anomalies + Backup + A-NotEnoughDC + The number of DCs is too small to provide redundancy: 1 DC + + + 5 + Anomalies + NetworkSniffing + A-HardenedPaths + Hardened Paths have been modified to lower the security level + + + 5 + StaleObjects + NetworkTopography + S-DC-SubnetMissing + The subnet declaration is incomplete [1 IP of DC not found in declared subnets] + + + 1 + StaleObjects + ObjectConfig + S-PwdNeverExpires + Number of accounts which have never expiring passwords: 1 + + + 0 + Anomalies + Reconnaissance + A-PreWin2000AuthenticatedUsers + The PreWin2000 compatible group contains "Authenticated Users" + + + 0 + Anomalies + Reconnaissance + A-DsHeuristicsLDAPSecurity + DsHeuristics has not been set to enable the mitigation for CVE-2019-12345 + + + 0 + PrivilegedAccounts + ACLCheck + P-DNSAdmin + Number of members of the Dns Admins group: 6 + + + 0 + Anomalies + NetworkSniffing + A-DnsZoneAUCreateChild + Authenticated Users can create DNS records + + + 0 + Anomalies + Audit + A-AuditPowershell + The PowerShell audit configuration is not fully enabled. + + + 0 + StaleObjects + ObjectConfig + S-KerberosArmoringDC + Verify Kerberos Armoring is enabled on DCs and the domain functional level is at least Windows Server 2012 + + + 0 + Anomalies + WeakPassword + A-NoServicePolicy + No password policy for service accounts found (MinimumPasswordLength>=20) + + + 0 + StaleObjects + ObjectConfig + S-KerberosArmoring + Verify Kerberos Armoring is enabled on clients and the domain functional level is at least Windows Server 2012 + + + 0 + PrivilegedAccounts + IrreversibleChange + P-UnprotectedOU + OU without the accidental deletion protection have been found + + + 0 + Anomalies + NetworkSniffing + A-NoGPOLLMNR + No GPO has been found which disables LLMNR or at least one GPO does enable it explicitly + + + 0 + Anomalies + Reconnaissance + A-NoNetSessionHardening + No GPO has been found which implements NetCease + + + + 105 + 104 + 1 + 104 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + + + 1 + 1 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + + + + Server OS + 1 + + 1 + 1 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + + + + + + + 1 + 1 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + + + + + + Default Domain Policy + {31B2F340-016D-11D2-945F-00C04FB984F9} + false + + DC=example,DC=local + + + 1 + + + + Default Domain Controllers Policy + {6AC1786C-016F-11D2-945F-00C04fB984F9} + false + + OU=Domain Controllers,DC=example,DC=local + + + 1 + + + + + + None + 104 + + + + 2024-06-06T10:04:32+02:00 + 9999-12-31T23:59:59.9999999 + 9999-12-31T23:59:59.9999999 + 0001-01-01T00:00:00 + 2024-03-13T01:25:35.9203718+01:00 + 2 + false + 2024-06-06T12:18:47.7308022+02:00 + false + + + + + + + + + + MinimumPasswordAge + 1 + + + MaximumPasswordAge + 42 + + + MinimumPasswordLength + 4 + + + PasswordComplexity + 0 + + + PasswordHistorySize + 24 + + + LockoutBadCount + 0 + + + ClearTextPassword + 0 + + + Default Domain Policy + {31B2F340-016D-11D2-945F-00C04FB984F9} + + + + + + + + + + + + + Account Operators + CN=Account Operators,CN=Builtin,DC=example,DC=local + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + Administrators + CN=Administrators,CN=Builtin,DC=example,DC=local + 1 + 0 + 0 + 1 + 0 + 0 + 1 + 1 + 1 + 0 + 0 + 0 + 0 + + + Backup Operators + CN=Backup Operators,CN=Builtin,DC=example,DC=local + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + Certificate Operators + CN=Cryptographic Operators,CN=Builtin,DC=example,DC=local + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + Certificate Publishers + CN=Cert Publishers,CN=Users,DC=example,DC=local + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + Dns Admins + CN=DnsAdmins,CN=Users,DC=example,DC=local + 6 + 0 + 0 + 0 + 0 + 6 + 0 + 6 + 6 + 0 + 0 + 0 + 0 + + + Domain Administrators + CN=Domain Admins,CN=Users,DC=example,DC=local + 1 + 0 + 0 + 1 + 0 + 0 + 1 + 1 + 1 + 0 + 0 + 0 + 0 + + + Enterprise Administrators + CN=Enterprise Admins,CN=Users,DC=example,DC=local + 1 + 0 + 0 + 1 + 0 + 0 + 1 + 1 + 1 + 0 + 0 + 0 + 0 + + + Enterprise Key Administrators + CN=Enterprise Key Admins,CN=Users,DC=example,DC=local + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + Key Administrators + CN=Key Admins,CN=Users,DC=example,DC=local + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + Print Operators + CN=Print Operators,CN=Builtin,DC=example,DC=local + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + Replicator + CN=Replicator,CN=Builtin,DC=example,DC=local + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + Schema Administrators + CN=Schema Admins,CN=Users,DC=example,DC=local + 1 + 0 + 0 + 1 + 0 + 0 + 1 + 1 + 1 + 0 + 0 + 0 + 0 + + + Server Operators + CN=Server Operators,CN=Builtin,DC=example,DC=local + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + false + 10 + + + + example.local + false + false + + + RootDNSServers + false + false + + + RootDNSServers + false + false + + + + + + + + + 0001-01-01T00:00:00 + 0 + 0 + + + + + + + + + false + + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 1 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 6 + 0 + 10 + 0 + + + + 1 + 0 + 0 + 0 + + + + 0 + 1 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 1 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 1 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 1 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + 0 + 0 + 0 + 0 + + + + + + + + + + + 0 + 0 + 0 + 0 + 0 + + \ No newline at end of file diff --git a/unittests/scans/pingcastle/one.xml b/unittests/scans/pingcastle/one.xml new file mode 100644 index 00000000000..d6b00890e38 --- /dev/null +++ b/unittests/scans/pingcastle/one.xml @@ -0,0 +1,32 @@ + + + 3.2.0.1 + 2024-06-06T13:01:09+02:00 + Normal + 2 + example.local + EXAMPLE + example.local + 2024-03-13T00:24:53 + S-1-5-21-1111111111-2222222222-3333333333 + 1 + 10 + + + DC-NQ7M31 + Server OS + + 198.51.10 + + + + + + 10 + Anomalies + WeakPassword + A-MinPwdLen + Password policy allows length less than 8 characters + + + \ No newline at end of file diff --git a/unittests/scans/pingcastle/zero.xml b/unittests/scans/pingcastle/zero.xml new file mode 100644 index 00000000000..21355251c96 --- /dev/null +++ b/unittests/scans/pingcastle/zero.xml @@ -0,0 +1,23 @@ + + + 3.2.0.1 + 2024-06-06T13:01:09+02:00 + Normal + 2 + example.local + EXAMPLE + example.local + 2024-03-13T00:24:53 + S-1-5-21-1111111111-2222222222-3333333333 + 1 + 10 + + + DC-NQ7M31 + Server OS + + 198.51.10 + + + + \ No newline at end of file diff --git a/unittests/scans/prowler/example_output_aws.csv b/unittests/scans/prowler/example_output_aws.csv new file mode 100644 index 00000000000..ebd601536c9 --- /dev/null +++ b/unittests/scans/prowler/example_output_aws.csv @@ -0,0 +1,5 @@ +AUTH_METHOD;TIMESTAMP;ACCOUNT_UID;ACCOUNT_NAME;ACCOUNT_EMAIL;ACCOUNT_ORGANIZATION_UID;ACCOUNT_ORGANIZATION_NAME;ACCOUNT_TAGS;FINDING_UID;PROVIDER;CHECK_ID;CHECK_TITLE;CHECK_TYPE;STATUS;STATUS_EXTENDED;MUTED;SERVICE_NAME;SUBSERVICE_NAME;SEVERITY;RESOURCE_TYPE;RESOURCE_UID;RESOURCE_NAME;RESOURCE_DETAILS;RESOURCE_TAGS;PARTITION;REGION;DESCRIPTION;RISK;RELATED_URL;REMEDIATION_RECOMMENDATION_TEXT;REMEDIATION_RECOMMENDATION_URL;REMEDIATION_CODE_NATIVEIAC;REMEDIATION_CODE_TERRAFORM;REMEDIATION_CODE_CLI;REMEDIATION_CODE_OTHER;COMPLIANCE;CATEGORIES;DEPENDS_ON;RELATED_TO;NOTES;PROWLER_VERSION;ADDITIONAL_URLS +;2025-02-14 14:27:03.913874;;;;;;;;aws;accessanalyzer_enabled;Check if IAM Access Analyzer is enabled;IAM;FAIL;IAM Access Analyzer in account is not enabled.;False;accessanalyzer;;low;Other;;;;;aws;;Check if IAM Access Analyzer is enabled;AWS IAM Access Analyzer helps you identify the resources in your organization and accounts, such as Amazon S3 buckets or IAM roles, that are shared with an external entity. This lets you identify unintended access to your resources and data, which is a security risk. IAM Access Analyzer uses a form of mathematical analysis called automated reasoning, which applies logic and mathematical inference to determine all possible access paths allowed by a resource policy.;https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html;Enable IAM Access Analyzer for all accounts, create analyzer and take action over it is recommendations (IAM Access Analyzer is available at no additional cost).;https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html;;;aws accessanalyzer create-analyzer --analyzer-name --type ;;CIS-1.4: 1.20 | CIS-1.5: 1.20 | KISA-ISMS-P-2023: 2.5.6, 2.6.4, 2.8.1, 2.8.2 | CIS-2.0: 1.20 | KISA-ISMS-P-2023-korean: 2.5.6, 2.6.4, 2.8.1, 2.8.2 | AWS-Account-Security-Onboarding: Enabled security services, Create analyzers in each active regions, Verify that events are present in SecurityHub aggregated view | CIS-3.0: 1.20;;;;;;https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-getting-started.html | https://aws.amazon.com/iam/features/analyze-access/ +;2025-02-14 14:27:03.913874;;;;;;;;aws;account_maintain_current_contact_details;Maintain current contact details.;IAM;MANUAL;Login to the AWS Console. Choose your account name on the top right of the window -> My Account -> Contact Information.;False;account;;medium;Other;;;;;aws;;Maintain current contact details.;Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization. An AWS account supports a number of contact details, and AWS will use these to contact the account owner if activity judged to be in breach of Acceptable Use Policy. If an AWS account is observed to be behaving in a prohibited or suspicious manner, AWS will attempt to contact the account owner by email and phone using the contact details listed. If this is unsuccessful and the account behavior needs urgent mitigation, proactive measures may be taken, including throttling of traffic between the account exhibiting suspicious behavior and the AWS API endpoints and the Internet. This will result in impaired service to and from the account in question.;;Using the Billing and Cost Management console complete contact details.;https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html;;;No command available.;https://docs.prowler.com/checks/aws/iam-policies/iam_18-maintain-contact-details#aws-console;CIS-1.4: 1.1 | CIS-1.5: 1.1 | KISA-ISMS-P-2023: 2.1.3 | CIS-2.0: 1.1 | KISA-ISMS-P-2023-korean: 2.1.3 | AWS-Well-Architected-Framework-Security-Pillar: SEC03-BP03, SEC10-BP01 | AWS-Account-Security-Onboarding: Billing, emergency, security contacts | CIS-3.0: 1.1 | ENS-RD2022: op.ext.7.aws.am.1;;;;;;https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-getting-started.html | https://aws.amazon.com/iam/features/analyze-access/ +;2025-02-14 14:27:03.913874;;;;;;;;aws;account_maintain_different_contact_details_to_security_billing_and_operations;Maintain different contact details to security, billing and operations.;IAM;FAIL;SECURITY, BILLING and OPERATIONS contacts not found or they are not different between each other and between ROOT contact.;False;account;;medium;Other;;;;;aws;;Maintain different contact details to security, billing and operations.;Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization. An AWS account supports a number of contact details, and AWS will use these to contact the account owner if activity judged to be in breach of Acceptable Use Policy. If an AWS account is observed to be behaving in a prohibited or suspicious manner, AWS will attempt to contact the account owner by email and phone using the contact details listed. If this is unsuccessful and the account behavior needs urgent mitigation, proactive measures may be taken, including throttling of traffic between the account exhibiting suspicious behavior and the AWS API endpoints and the Internet. This will result in impaired service to and from the account in question.;https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html;Using the Billing and Cost Management console complete contact details.;https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html;;;;https://docs.prowler.com/checks/aws/iam-policies/iam_18-maintain-contact-details#aws-console;KISA-ISMS-P-2023: 2.1.3 | KISA-ISMS-P-2023-korean: 2.1.3;;;;;;https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-getting-started.html | https://aws.amazon.com/iam/features/analyze-access/ +;2025-02-14 14:27:03.913874;;;;;;;;aws;account_security_contact_information_is_registered;Ensure security contact information is registered.;IAM;MANUAL;Login to the AWS Console. Choose your account name on the top right of the window -> My Account -> Alternate Contacts -> Security Section.;False;account;;medium;Other;:root;;;;aws;;Ensure security contact information is registered.;AWS provides customers with the option of specifying the contact information for accounts security team. It is recommended that this information be provided. Specifying security-specific contact information will help ensure that security advisories sent by AWS reach the team in your organization that is best equipped to respond to them.;;Go to the My Account section and complete alternate contacts.;https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html;;;No command available.;https://docs.prowler.com/checks/aws/iam-policies/iam_19#aws-console;CIS-1.4: 1.2 | CIS-1.5: 1.2 | AWS-Foundational-Security-Best-Practices: account, acm | KISA-ISMS-P-2023: 2.1.3, 2.2.1 | CIS-2.0: 1.2 | KISA-ISMS-P-2023-korean: 2.1.3, 2.2.1 | AWS-Well-Architected-Framework-Security-Pillar: SEC03-BP03, SEC10-BP01 | AWS-Account-Security-Onboarding: Billing, emergency, security contacts | CIS-3.0: 1.2 | ENS-RD2022: op.ext.7.aws.am.1;;;;;;https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-getting-started.html | https://aws.amazon.com/iam/features/analyze-access/ diff --git a/unittests/scans/prowler/example_output_aws.ocsf.json b/unittests/scans/prowler/example_output_aws.ocsf.json new file mode 100644 index 00000000000..0de6818e2fb --- /dev/null +++ b/unittests/scans/prowler/example_output_aws.ocsf.json @@ -0,0 +1,369 @@ +[ + { + "message": "IAM Access Analyzer in account is not enabled.", + "metadata": { + "event_code": "accessanalyzer_enabled", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "" + }, + "profiles": [ + "cloud", + "datetime" + ], + "tenant_uid": "", + "version": "1.4.0" + }, + "severity_id": 2, + "severity": "Low", + "status": "New", + "status_code": "FAIL", + "status_detail": "IAM Access Analyzer in account is not enabled.", + "status_id": 1, + "unmapped": { + "related_url": "https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html", + "categories": [], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "", + "compliance": { + "CIS-1.4": [ + "1.20" + ], + "CIS-1.5": [ + "1.20" + ], + "KISA-ISMS-P-2023": [ + "2.5.6", + "2.6.4", + "2.8.1", + "2.8.2" + ], + "CIS-2.0": [ + "1.20" + ], + "KISA-ISMS-P-2023-korean": [ + "2.5.6", + "2.6.4", + "2.8.1", + "2.8.2" + ], + "AWS-Account-Security-Onboarding": [ + "Enabled security services", + "Create analyzers in each active regions", + "Verify that events are present in SecurityHub aggregated view" + ], + "CIS-3.0": [ + "1.20" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539623, + "created_time_dt": "2025-02-14T14:27:03.913874", + "desc": "Check if IAM Access Analyzer is enabled", + "product_uid": "prowler", + "title": "Check if IAM Access Analyzer is enabled", + "types": [ + "IAM" + ], + "uid": "" + }, + "resources": [ + { + "cloud_partition": "aws", + "region": "", + "data": { + "details": "", + "metadata": { + "arn": "", + "name": "", + "status": "NOT_AVAILABLE", + "findings": [], + "tags": [], + "type": "", + "region": "" + } + }, + "group": { + "name": "accessanalyzer" + }, + "labels": [], + "name": "", + "type": "Other", + "uid": "" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "cloud": { + "account": { + "name": "", + "type": "AWS Account", + "type_id": 10, + "uid": "", + "labels": [] + }, + "org": { + "name": "", + "uid": "" + }, + "provider": "aws", + "region": "" + }, + "remediation": { + "desc": "Enable IAM Access Analyzer for all accounts, create analyzer and take action over it is recommendations (IAM Access Analyzer is available at no additional cost).", + "references": [ + "aws accessanalyzer create-analyzer --analyzer-name --type ", + "https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html" + ] + }, + "risk_details": "AWS IAM Access Analyzer helps you identify the resources in your organization and accounts, such as Amazon S3 buckets or IAM roles, that are shared with an external entity. This lets you identify unintended access to your resources and data, which is a security risk. IAM Access Analyzer uses a form of mathematical analysis called automated reasoning, which applies logic and mathematical inference to determine all possible access paths allowed by a resource policy.", + "time": 1739539623, + "time_dt": "2025-02-14T14:27:03.913874", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + }, + { + "message": "Login to the AWS Console. Choose your account name on the top right of the window -> My Account -> Contact Information.", + "metadata": { + "event_code": "account_maintain_current_contact_details", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "" + }, + "profiles": [ + "cloud", + "datetime" + ], + "tenant_uid": "", + "version": "1.4.0" + }, + "severity_id": 3, + "severity": "Medium", + "status": "New", + "status_code": "MANUAL", + "status_detail": "Login to the AWS Console. Choose your account name on the top right of the window -> My Account -> Contact Information.", + "status_id": 1, + "unmapped": { + "related_url": "", + "categories": [], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "", + "compliance": { + "CIS-1.4": [ + "1.1" + ], + "CIS-1.5": [ + "1.1" + ], + "KISA-ISMS-P-2023": [ + "2.1.3" + ], + "CIS-2.0": [ + "1.1" + ], + "KISA-ISMS-P-2023-korean": [ + "2.1.3" + ], + "AWS-Well-Architected-Framework-Security-Pillar": [ + "SEC03-BP03", + "SEC10-BP01" + ], + "AWS-Account-Security-Onboarding": [ + "Billing, emergency, security contacts" + ], + "CIS-3.0": [ + "1.1" + ], + "ENS-RD2022": [ + "op.ext.7.aws.am.1" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539623, + "created_time_dt": "2025-02-14T14:27:03.913874", + "desc": "Maintain current contact details.", + "product_uid": "prowler", + "title": "Maintain current contact details.", + "types": [ + "IAM" + ], + "uid": "" + }, + "resources": [ + { + "cloud_partition": "aws", + "region": "", + "data": { + "details": "", + "metadata": { + "type": "PRIMARY", + "email": null, + "name": "", + "phone_number": "" + } + }, + "group": { + "name": "account" + }, + "labels": [], + "name": "", + "type": "Other", + "uid": "arn:aws:iam:::root" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "cloud": { + "account": { + "name": "", + "type": "AWS Account", + "type_id": 10, + "uid": "", + "labels": [] + }, + "org": { + "name": "", + "uid": "" + }, + "provider": "aws", + "region": "" + }, + "remediation": { + "desc": "Using the Billing and Cost Management console complete contact details.", + "references": [ + "No command available.", + "https://docs.prowler.com/checks/aws/iam-policies/iam_18-maintain-contact-details#aws-console", + "https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html" + ] + }, + "risk_details": "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization. An AWS account supports a number of contact details, and AWS will use these to contact the account owner if activity judged to be in breach of Acceptable Use Policy. If an AWS account is observed to be behaving in a prohibited or suspicious manner, AWS will attempt to contact the account owner by email and phone using the contact details listed. If this is unsuccessful and the account behavior needs urgent mitigation, proactive measures may be taken, including throttling of traffic between the account exhibiting suspicious behavior and the AWS API endpoints and the Internet. This will result in impaired service to and from the account in question.", + "time": 1739539623, + "time_dt": "2025-02-14T14:27:03.913874", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + }, + { + "message": "SECURITY, BILLING and OPERATIONS contacts not found or they are not different between each other and between ROOT contact.", + "metadata": { + "event_code": "account_maintain_different_contact_details_to_security_billing_and_operations", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "" + }, + "profiles": [ + "cloud", + "datetime" + ], + "tenant_uid": "", + "version": "1.4.0" + }, + "severity_id": 3, + "severity": "Medium", + "status": "New", + "status_code": "FAIL", + "status_detail": "SECURITY, BILLING and OPERATIONS contacts not found or they are not different between each other and between ROOT contact.", + "status_id": 1, + "unmapped": { + "related_url": "https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html", + "categories": [], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "", + "compliance": { + "KISA-ISMS-P-2023": [ + "2.1.3" + ], + "KISA-ISMS-P-2023-korean": [ + "2.1.3" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539623, + "created_time_dt": "2025-02-14T14:27:03.913874", + "desc": "Maintain different contact details to security, billing and operations.", + "product_uid": "prowler", + "title": "Maintain different contact details to security, billing and operations.", + "types": [ + "IAM" + ], + "uid": "" + }, + "resources": [ + { + "cloud_partition": "aws", + "region": "", + "data": { + "details": "", + "metadata": { + "type": "PRIMARY", + "email": null, + "name": "", + "phone_number": "" + } + }, + "group": { + "name": "account" + }, + "labels": [], + "name": "", + "type": "Other", + "uid": "arn:aws:iam:::root" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "cloud": { + "account": { + "name": "", + "type": "AWS Account", + "type_id": 10, + "uid": "", + "labels": [] + }, + "org": { + "name": "", + "uid": "" + }, + "provider": "aws", + "region": "" + }, + "remediation": { + "desc": "Using the Billing and Cost Management console complete contact details.", + "references": [ + "https://docs.prowler.com/checks/aws/iam-policies/iam_18-maintain-contact-details#aws-console", + "https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html" + ] + }, + "risk_details": "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization. An AWS account supports a number of contact details, and AWS will use these to contact the account owner if activity judged to be in breach of Acceptable Use Policy. If an AWS account is observed to be behaving in a prohibited or suspicious manner, AWS will attempt to contact the account owner by email and phone using the contact details listed. If this is unsuccessful and the account behavior needs urgent mitigation, proactive measures may be taken, including throttling of traffic between the account exhibiting suspicious behavior and the AWS API endpoints and the Internet. This will result in impaired service to and from the account in question.", + "time": 1739539623, + "time_dt": "2025-02-14T14:27:03.913874", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + } +] diff --git a/unittests/scans/prowler/example_output_azure.csv b/unittests/scans/prowler/example_output_azure.csv new file mode 100644 index 00000000000..624ecfd2f2e --- /dev/null +++ b/unittests/scans/prowler/example_output_azure.csv @@ -0,0 +1,5 @@ +AUTH_METHOD;TIMESTAMP;ACCOUNT_UID;ACCOUNT_NAME;ACCOUNT_EMAIL;ACCOUNT_ORGANIZATION_UID;ACCOUNT_ORGANIZATION_NAME;ACCOUNT_TAGS;FINDING_UID;PROVIDER;CHECK_ID;CHECK_TITLE;CHECK_TYPE;STATUS;STATUS_EXTENDED;MUTED;SERVICE_NAME;SUBSERVICE_NAME;SEVERITY;RESOURCE_TYPE;RESOURCE_UID;RESOURCE_NAME;RESOURCE_DETAILS;RESOURCE_TAGS;PARTITION;REGION;DESCRIPTION;RISK;RELATED_URL;REMEDIATION_RECOMMENDATION_TEXT;REMEDIATION_RECOMMENDATION_URL;REMEDIATION_CODE_NATIVEIAC;REMEDIATION_CODE_TERRAFORM;REMEDIATION_CODE_CLI;REMEDIATION_CODE_OTHER;COMPLIANCE;CATEGORIES;DEPENDS_ON;RELATED_TO;NOTES;PROWLER_VERSION;ADDITIONAL_URLS +;2025-02-14 14:27:30.710664;;;;;ProwlerPro.onmicrosoft.com;;;azure;aks_cluster_rbac_enabled;Ensure AKS RBAC is enabled;;FAIL;RBAC is enabled for cluster '' in subscription ''.;False;aks;;medium;Microsoft.ContainerService/ManagedClusters;/subscriptions//resourcegroups/_group/providers/Microsoft.ContainerService/managedClusters/;;;;;;Azure Kubernetes Service (AKS) can be configured to use Azure Active Directory (AD) for user authentication. In this configuration, you sign in to an AKS cluster using an Azure AD authentication token. You can also configure Kubernetes role-based access control (Kubernetes RBAC) to limit access to cluster resources based a user's identity or group membership.;Kubernetes RBAC and AKS help you secure your cluster access and provide only the minimum required permissions to developers and operators.;https://learn.microsoft.com/en-us/azure/aks/azure-ad-rbac?tabs=portal;;https://learn.microsoft.com/en-us/security/benchmark/azure/security-controls-v2-privileged-access#pa-7-follow-just-enough-administration-least-privilege-principle;;https://docs.prowler.com/checks/azure/azure-kubernetes-policies/bc_azr_kubernetes_2#terraform;;https://www.trendmicro.com/cloudoneconformity/knowledge-base/azure/AKS/enable-role-based-access-control-for-kubernetes-service.html#;ENS-RD2022: op.acc.2.az.r1.eid.1;;;;;;https://learn.microsoft.com/azure/aks/azure-ad-rbac | https://learn.microsoft.com/azure/aks/concepts-identity +;2025-02-14 14:27:30.710664;;;;;ProwlerPro.onmicrosoft.com;;;azure;aks_clusters_created_with_private_nodes;Ensure clusters are created with Private Nodes;;FAIL;Cluster '' was created with private nodes in subscription '';False;aks;;high;Microsoft.ContainerService/ManagedClusters;/subscriptions//resourcegroups/_group/providers/Microsoft.ContainerService/managedClusters/;;;;;;Disable public IP addresses for cluster nodes, so that they only have private IP addresses. Private Nodes are nodes with no public IP addresses.;Disabling public IP addresses on cluster nodes restricts access to only internal networks, forcing attackers to obtain local network access before attempting to compromise the underlying Kubernetes hosts.;https://learn.microsoft.com/en-us/azure/aks/private-clusters;;https://learn.microsoft.com/en-us/azure/aks/access-private-cluster;;;;;ENS-RD2022: mp.com.4.r2.az.aks.1 | MITRE-ATTACK: T1190, T1530;;;;;;https://learn.microsoft.com/azure/aks/azure-ad-rbac | https://learn.microsoft.com/azure/aks/concepts-identity +;2025-02-14 14:27:30.710664;;;;;ProwlerPro.onmicrosoft.com;;;azure;aks_clusters_public_access_disabled;Ensure clusters are created with Private Endpoint Enabled and Public Access Disabled;;FAIL;Public access to nodes is enabled for cluster '' in subscription '';False;aks;;high;Microsoft.ContainerService/ManagedClusters;/subscriptions//resourcegroups/_group/providers/Microsoft.ContainerService/managedClusters/;;;;;;Disable access to the Kubernetes API from outside the node network if it is not required.;In a private cluster, the master node has two endpoints, a private and public endpoint. The private endpoint is the internal IP address of the master, behind an internal load balancer in the master's wirtual network. Nodes communicate with the master using the private endpoint. The public endpoint enables the Kubernetes API to be accessed from outside the master's virtual network. Although Kubernetes API requires an authorized token to perform sensitive actions, a vulnerability could potentially expose the Kubernetes publically with unrestricted access. Additionally, an attacker may be able to identify the current cluster and Kubernetes API version and determine whether it is vulnerable to an attack. Unless required, disabling public endpoint will help prevent such threats, and require the attacker to be on the master's virtual network to perform any attack on the Kubernetes API.;https://learn.microsoft.com/en-us/azure/aks/private-clusters?tabs=azure-portal;To use a private endpoint, create a new private endpoint in your virtual network then create a link between your virtual network and a new private DNS zone;https://learn.microsoft.com/en-us/azure/aks/access-private-cluster?tabs=azure-cli;;;az aks update -n -g --disable-public-fqdn;;ENS-RD2022: mp.com.4.az.aks.2 | MITRE-ATTACK: T1190, T1530;;;;;;https://learn.microsoft.com/azure/aks/azure-ad-rbac | https://learn.microsoft.com/azure/aks/concepts-identity +;2025-02-14 14:27:30.710664;;;;;ProwlerPro.onmicrosoft.com;;;azure;aks_network_policy_enabled;Ensure Network Policy is Enabled and set as appropriate;;FAIL;Network policy is enabled for cluster '' in subscription ''.;False;aks;;medium;Microsoft.ContainerService/managedClusters;/subscriptions//resourcegroups/_group/providers/Microsoft.ContainerService/managedClusters/;;;;;;When you run modern, microservices-based applications in Kubernetes, you often want to control which components can communicate with each other. The principle of least privilege should be applied to how traffic can flow between pods in an Azure Kubernetes Service (AKS) cluster. Let's say you likely want to block traffic directly to back-end applications. The Network Policy feature in Kubernetes lets you define rules for ingress and egress traffic between pods in a cluster.;All pods in an AKS cluster can send and receive traffic without limitations, by default. To improve security, you can define rules that control the flow of traffic. Back-end applications are often only exposed to required front-end services, for example. Or, database components are only accessible to the application tiers that connect to them. Network Policy is a Kubernetes specification that defines access policies for communication between Pods. Using Network Policies, you define an ordered set of rules to send and receive traffic and apply them to a collection of pods that match one or more label selectors. These network policy rules are defined as YAML manifests. Network policies can be included as part of a wider manifest that also creates a deployment or service.;https://learn.microsoft.com/en-us/security/benchmark/azure/security-controls-v2-network-security#ns-2-connect-private-networks-together;;https://learn.microsoft.com/en-us/azure/aks/use-network-policies;;https://docs.prowler.com/checks/azure/azure-kubernetes-policies/bc_azr_kubernetes_4#terraform;;;ENS-RD2022: mp.com.4.r2.az.aks.1;;;;Network Policy requires the Network Policy add-on. This add-on is included automatically when a cluster with Network Policy is created, but for an existing cluster, needs to be added prior to enabling Network Policy. Enabling/Disabling Network Policy causes a rolling update of all cluster nodes, similar to performing a cluster upgrade. This operation is long-running and will block other operations on the cluster (including delete) until it has run to completion. If Network Policy is used, a cluster must have at least 2 nodes of type n1-standard-1 or higher. The recommended minimum size cluster to run Network Policy enforcement is 3 n1-standard-1 instances. Enabling Network Policy enforcement consumes additional resources in nodes. Specifically, it increases the memory footprint of the kube-system process by approximately 128MB, and requires approximately 300 millicores of CPU.;;https://learn.microsoft.com/azure/aks/azure-ad-rbac | https://learn.microsoft.com/azure/aks/concepts-identity diff --git a/unittests/scans/prowler/example_output_azure.ocsf.json b/unittests/scans/prowler/example_output_azure.ocsf.json new file mode 100644 index 00000000000..9c916ef7580 --- /dev/null +++ b/unittests/scans/prowler/example_output_azure.ocsf.json @@ -0,0 +1,342 @@ +[ + { + "message": "There are no AppInsight configured in subscription .", + "metadata": { + "event_code": "appinsights_ensure_is_configured", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "5.4.0" + }, + "profiles": [ + "cloud", + "datetime" + ], + "tenant_uid": "", + "version": "1.4.0" + }, + "severity_id": 2, + "severity": "Low", + "status": "New", + "status_code": "FAIL", + "status_detail": "There are no AppInsight configured in subscription .", + "status_id": 1, + "unmapped": { + "related_url": "https://learn.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview", + "categories": [], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "Because Application Insights relies on a Log Analytics Workspace, an organization will incur additional expenses when using this service.", + "compliance": { + "CIS-2.1": [ + "5.3.1" + ], + "ENS-RD2022": [ + "mp.s.4.r1.az.nt.2" + ], + "CIS-3.0": [ + "6.3.1" + ], + "CIS-2.0": [ + "5.3.1" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539650, + "created_time_dt": "2025-02-14T14:27:30.710664", + "desc": "Application Insights within Azure act as an Application Performance Monitoring solution providing valuable data into how well an application performs and additional information when performing incident response. The types of log data collected include application metrics, telemetry data, and application trace logging data providing organizations with detailed information about application activity and application transactions. Both data sets help organizations adopt a proactive and retroactive means to handle security and performance related metrics within their modern applications.", + "product_uid": "prowler", + "title": "Ensure Application Insights are Configured.", + "types": [], + "uid": "" + }, + "resources": [ + { + "cloud_partition": "AzureCloud", + "region": "global", + "data": { + "details": "", + "metadata": {} + }, + "group": { + "name": "appinsights" + }, + "labels": [], + "name": "AppInsights", + "type": "Microsoft.Insights/components", + "uid": "AppInsights" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "cloud": { + "account": { + "name": "", + "type": "Azure AD Account", + "type_id": 6, + "uid": "", + "labels": [] + }, + "org": { + "name": "", + "uid": "" + }, + "provider": "azure", + "region": "global" + }, + "remediation": { + "desc": "1. Navigate to Application Insights 2. Under the Basics tab within the PROJECT DETAILS section, select the Subscription 3. Select the Resource group 4. Within the INSTANCE DETAILS, enter a Name 5. Select a Region 6. Next to Resource Mode, select Workspace-based 7. Within the WORKSPACE DETAILS, select the Subscription for the log analytics workspace 8. Select the appropriate Log Analytics Workspace 9. Click Next:Tags > 10. Enter the appropriate Tags as Name, Value pairs. 11. Click Next:Review+Create 12. Click Create.", + "references": [ + "az monitor app-insights component create --app --resource-group --location --kind 'web' --retention-time --workspace -- subscription ", + "https://www.tenable.com/audits/items/CIS_Microsoft_Azure_Foundations_v2.0.0_L2.audit:8a7a608d180042689ad9d3f16aa359f1" + ] + }, + "risk_details": "Configuring Application Insights provides additional data not found elsewhere within Azure as part of a much larger logging and monitoring program within an organization's Information Security practice. The types and contents of these logs will act as both a potential cost saving measure (application performance) and a means to potentially confirm the source of a potential incident (trace logging). Metrics and Telemetry data provide organizations with a proactive approach to cost savings by monitoring an application's performance, while the trace logging data provides necessary details in a reactive incident response scenario by helping organizations identify the potential source of an incident within their application.", + "time": 1739539650, + "time_dt": "2025-02-14T14:27:30.710664", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + }, + { + "message": "There is not another correct email configured for subscription .", + "metadata": { + "event_code": "defender_additional_email_configured_with_a_security_contact", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "5.4.0" + }, + "profiles": [ + "cloud", + "datetime" + ], + "tenant_uid": "", + "version": "1.4.0" + }, + "severity_id": 3, + "severity": "Medium", + "status": "New", + "status_code": "FAIL", + "status_detail": "There is not another correct email configured for subscription .", + "status_id": 1, + "unmapped": { + "related_url": "https://docs.microsoft.com/en-us/azure/security-center/security-center-provide-security-contact-details", + "categories": [], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "", + "compliance": { + "CIS-2.1": [ + "2.1.18" + ], + "ENS-RD2022": [ + "op.mon.3.r3.az.de.1" + ], + "CIS-3.0": [ + "3.1.13" + ], + "CIS-2.0": [ + "2.1.19" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539650, + "created_time_dt": "2025-02-14T14:27:30.710664", + "desc": "Microsoft Defender for Cloud emails the subscription owners whenever a high-severity alert is triggered for their subscription. You should provide a security contact email address as an additional email address.", + "product_uid": "prowler", + "title": "Ensure 'Additional email addresses' is Configured with a Security Contact Email", + "types": [], + "uid": "" + }, + "resources": [ + { + "cloud_partition": "AzureCloud", + "region": "global", + "data": { + "details": "", + "metadata": { + "resource_id": "", + "name": "", + "emails": "", + "phone": "", + "alert_notifications_minimal_severity": "High", + "alert_notifications_state": "On", + "notified_roles": [ + "Owner" + ], + "notified_roles_state": "On" + } + }, + "group": { + "name": "defender" + }, + "labels": [], + "name": "", + "type": "AzureEmailNotifications", + "uid": "" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "cloud": { + "account": { + "name": "", + "type": "Azure AD Account", + "type_id": 6, + "uid": "", + "labels": [] + }, + "org": { + "name": "", + "uid": "" + }, + "provider": "azure", + "region": "global" + }, + "remediation": { + "desc": "1. From Azure Home select the Portal Menu 2. Select Microsoft Defender for Cloud 3. Click on Environment Settings 4. Click on the appropriate Management Group, Subscription, or Workspace 5. Click on Email notifications 6. Enter a valid security contact email address (or multiple addresses separated by commas) in the Additional email addresses field 7. Click Save", + "references": [ + "https://docs.prowler.com/checks/azure/azure-general-policies/ensure-that-security-contact-emails-is-set#terraform", + "https://www.trendmicro.com/cloudoneconformity/knowledge-base/azure/SecurityCenter/security-contact-email.html", + "https://learn.microsoft.com/en-us/rest/api/defenderforcloud/security-contacts/list?view=rest-defenderforcloud-2020-01-01-preview&tabs=HTTP" + ] + }, + "risk_details": "Microsoft Defender for Cloud emails the Subscription Owner to notify them about security alerts. Adding your Security Contact's email address to the 'Additional email addresses' field ensures that your organization's Security Team is included in these alerts. This ensures that the proper people are aware of any potential compromise in order to mitigate the risk in a timely fashion.", + "time": 1739539650, + "time_dt": "2025-02-14T14:27:30.710664", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + }, + { + "message": "Defender plan Defender for App Services from subscription is set to OFF (pricing tier not standard).", + "metadata": { + "event_code": "defender_ensure_defender_for_app_services_is_on", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "5.4.0" + }, + "profiles": [ + "cloud", + "datetime" + ], + "tenant_uid": "", + "version": "1.4.0" + }, + "severity_id": 4, + "severity": "High", + "status": "New", + "status_code": "FAIL", + "status_detail": "Defender plan Defender for App Services from subscription is set to OFF (pricing tier not standard).", + "status_id": 1, + "unmapped": { + "related_url": "", + "categories": [], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "", + "compliance": { + "CIS-2.1": [ + "2.1.2" + ], + "ENS-RD2022": [ + "mp.s.4.r1.az.nt.3" + ], + "MITRE-ATTACK": [ + "T1190", + "T1059", + "T1204", + "T1552", + "T1486", + "T1499", + "T1496", + "T1087" + ], + "CIS-3.0": [ + "3.1.6.1" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539650, + "created_time_dt": "2025-02-14T14:27:30.710664", + "desc": "Ensure That Microsoft Defender for App Services Is Set To 'On' ", + "product_uid": "prowler", + "title": "Ensure That Microsoft Defender for App Services Is Set To 'On' ", + "types": [], + "uid": "" + }, + "resources": [ + { + "cloud_partition": "AzureCloud", + "region": "global", + "data": { + "details": "", + "metadata": { + "resource_id": "", + "pricing_tier": "Free", + "free_trial_remaining_time": 2592000.0, + "extensions": {} + } + }, + "group": { + "name": "defender" + }, + "labels": [], + "name": "", + "type": "AzureDefenderPlan", + "uid": "" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "cloud": { + "account": { + "name": "", + "type": "Azure AD Account", + "type_id": 6, + "uid": "", + "labels": [] + }, + "org": { + "name": "", + "uid": "" + }, + "provider": "azure", + "region": "global" + }, + "remediation": { + "desc": "By , Microsoft Defender for Cloud is not enabled for your App Service instances. Enabling the Defender security service for App Service instances allows for advanced security defense using threat detection capabilities provided by Microsoft Security Response Center.", + "references": [ + "https://docs.prowler.com/checks/azure/azure-general-policies/ensure-that-azure-defender-is-set-to-on-for-app-service#terraform", + "https://www.trendmicro.com/cloudoneconformity/knowledge-base/azure/SecurityCenter/defender-app-service.html", + "https://www.trendmicro.com/cloudoneconformity/knowledge-base/azure/SecurityCenter/defender-app-service.html" + ] + }, + "risk_details": "Turning on Microsoft Defender for App Service enables threat detection for App Service, providing threat intelligence, anomaly detection, and behavior analytics in the Microsoft Defender for Cloud.", + "time": 1739539650, + "time_dt": "2025-02-14T14:27:30.710664", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + } +] diff --git a/unittests/scans/prowler/example_output_gcp.csv b/unittests/scans/prowler/example_output_gcp.csv new file mode 100644 index 00000000000..37e5c00f170 --- /dev/null +++ b/unittests/scans/prowler/example_output_gcp.csv @@ -0,0 +1,5 @@ +AUTH_METHOD;TIMESTAMP;ACCOUNT_UID;ACCOUNT_NAME;ACCOUNT_EMAIL;ACCOUNT_ORGANIZATION_UID;ACCOUNT_ORGANIZATION_NAME;ACCOUNT_TAGS;FINDING_UID;PROVIDER;CHECK_ID;CHECK_TITLE;CHECK_TYPE;STATUS;STATUS_EXTENDED;MUTED;SERVICE_NAME;SUBSERVICE_NAME;SEVERITY;RESOURCE_TYPE;RESOURCE_UID;RESOURCE_NAME;RESOURCE_DETAILS;RESOURCE_TAGS;PARTITION;REGION;DESCRIPTION;RISK;RELATED_URL;REMEDIATION_RECOMMENDATION_TEXT;REMEDIATION_RECOMMENDATION_URL;REMEDIATION_CODE_NATIVEIAC;REMEDIATION_CODE_TERRAFORM;REMEDIATION_CODE_CLI;REMEDIATION_CODE_OTHER;COMPLIANCE;CATEGORIES;DEPENDS_ON;RELATED_TO;NOTES;PROWLER_VERSION;ADDITIONAL_URLS +;2025-02-14 14:27:20.697446;;;;;;;;gcp;apikeys_key_exists;Ensure API Keys Only Exist for Active Services;;PASS;Project does not have active API Keys.;False;apikeys;;medium;API Key;;;;;;;API Keys should only be used for services in cases where other authentication methods are unavailable. Unused keys with their permissions in tact may still exist within a project. Keys are insecure because they can be viewed publicly, such as from within a browser, or they can be accessed on a device where the key resides. It is recommended to use standard authentication flow instead.;Security risks involved in using API-Keys appear below: API keys are simple encrypted strings, API keys do not identify the user or the application making the API request, API keys are typically accessible to clients, making it easy to discover and steal an API key.;;To avoid the security risk in using API keys, it is recommended to use standard authentication flow instead.;https://cloud.google.com/docs/authentication/api-keys;;;gcloud alpha services api-keys delete;;MITRE-ATTACK: T1098 | CIS-2.0: 1.12 | ENS-RD2022: op.acc.2.gcp.rbak.1 | CIS-3.0: 1.12;;;;;;https://cloud.google.com/api-keys/docs/best-practices | https://cloud.google.com/docs/authentication +;2025-02-14 14:27:20.697446;;;;;;;;gcp;artifacts_container_analysis_enabled;Ensure Image Vulnerability Analysis using AR Container Analysis or a third-party provider;Security | Configuration;FAIL;AR Container Analysis is not enabled in project .;False;artifacts;Container Analysis;medium;Service;;;;;;;Scan images stored in Google Container Registry (GCR) for vulnerabilities using AR Container Analysis or a third-party provider. This helps identify and mitigate security risks associated with known vulnerabilities in container images.;Without image vulnerability scanning, container images stored in Artifact Registry may contain known vulnerabilities, increasing the risk of exploitation by malicious actors.;https://cloud.google.com/artifact-analysis/docs;Enable vulnerability scanning for images stored in Artifact Registry using AR Container Analysis or a third-party provider.;https://cloud.google.com/artifact-analysis/docs/container-scanning-overview;;;gcloud services enable containeranalysis.googleapis.com;;MITRE-ATTACK: T1525 | ENS-RD2022: op.exp.4.r4.gcp.log.1, op.mon.3.gcp.scc.1;;;;By default, AR Container Analysis is disabled.;;https://cloud.google.com/api-keys/docs/best-practices | https://cloud.google.com/docs/authentication +;2025-02-14 14:27:20.697446;;;;;;;;gcp;compute_firewall_rdp_access_from_the_internet_allowed;Ensure That RDP Access Is Restricted From the Internet;;FAIL;Firewall does not expose port 3389 (RDP) to the internet.;False;networking;;critical;FirewallRule;;;;;;;GCP `Firewall Rules` are specific to a `VPC Network`. Each rule either `allows` or `denies` traffic when its conditions are met. Its conditions allow users to specify the type of traffic, such as ports and protocols, and the source or destination of the traffic, including IP addresses, subnets, and instances. Firewall rules are defined at the VPC network level and are specific to the network in which they are defined. The rules themselves cannot be shared among networks. Firewall rules only support IPv4 traffic. When specifying a source for an ingress rule or a destination for an egress rule by address, an `IPv4` address or `IPv4 block in CIDR` notation can be used. Generic `(0.0.0.0/0)` incoming traffic from the Internet to a VPC or VM instance using `RDP` on `Port 3389` can be avoided.;Allowing unrestricted Remote Desktop Protocol (RDP) access can increase opportunities for malicious activities such as hacking, Man-In-The-Middle attacks (MITM) and Pass-The-Hash (PTH) attacks.;;Ensure that Google Cloud Virtual Private Cloud (VPC) firewall rules do not allow unrestricted access (i.e. 0.0.0.0/0) on TCP port 3389 in order to restrict Remote Desktop Protocol (RDP) traffic to trusted IP addresses or IP ranges only and reduce the attack surface. TCP port 3389 is used for secure remote GUI login to Windows VM instances by connecting a RDP client application with an RDP server.;https://cloud.google.com/vpc/docs/using-firewalls;;https://docs./checks/gcp/google-cloud-networking-policies/bc_gcp_networking_2#terraform;https://docs./checks/gcp/google-cloud-networking-policies/bc_gcp_networking_2#cli-command;https://www.trendmicro.com/cloudoneconformity/knowledge-base/gcp/CloudVPC/unrestricted-rdp-access.html;MITRE-ATTACK: T1190, T1199, T1048, T1498, T1046 | CIS-2.0: 3.7 | ENS-RD2022: mp.com.1.gcp.fw.1 | CIS-3.0: 3.7;internet-exposed;;;;;https://cloud.google.com/api-keys/docs | https://cloud.google.com/docs/authentication +;2025-02-14 14:27:20.697446;;;;;;;;gcp;compute_firewall_rdp_access_from_the_internet_allowed;Ensure That RDP Access Is Restricted From the Internet;;FAIL;Firewall does not expose port 3389 (RDP) to the internet.;False;networking;;critical;FirewallRule;;;;;;;GCP `Firewall Rules` are specific to a `VPC Network`. Each rule either `allows` or `denies` traffic when its conditions are met. Its conditions allow users to specify the type of traffic, such as ports and protocols, and the source or destination of the traffic, including IP addresses, subnets, and instances. Firewall rules are defined at the VPC network level and are specific to the network in which they are defined. The rules themselves cannot be shared among networks. Firewall rules only support IPv4 traffic. When specifying a source for an ingress rule or a destination for an egress rule by address, an `IPv4` address or `IPv4 block in CIDR` notation can be used. Generic `(0.0.0.0/0)` incoming traffic from the Internet to a VPC or VM instance using `RDP` on `Port 3389` can be avoided.;Allowing unrestricted Remote Desktop Protocol (RDP) access can increase opportunities for malicious activities such as hacking, Man-In-The-Middle attacks (MITM) and Pass-The-Hash (PTH) attacks.;;Ensure that Google Cloud Virtual Private Cloud (VPC) firewall rules do not allow unrestricted access (i.e. 0.0.0.0/0) on TCP port 3389 in order to restrict Remote Desktop Protocol (RDP) traffic to trusted IP addresses or IP ranges only and reduce the attack surface. TCP port 3389 is used for secure remote GUI login to Windows VM instances by connecting a RDP client application with an RDP server.;https://cloud.google.com/vpc/docs/using-firewalls;;https://docs./checks/gcp/google-cloud-networking-policies/bc_gcp_networking_2#terraform;https://docs./checks/gcp/google-cloud-networking-policies/bc_gcp_networking_2#cli-command;https://www.trendmicro.com/cloudoneconformity/knowledge-base/gcp/CloudVPC/unrestricted-rdp-access.html;MITRE-ATTACK: T1190, T1199, T1048, T1498, T1046 | CIS-2.0: 3.7 | ENS-RD2022: mp.com.1.gcp.fw.1 | CIS-3.0: 3.7;internet-exposed;;;;;https://cloud.google.com/api-keys/docs | https://cloud.google.com/docs/authentication diff --git a/unittests/scans/prowler/example_output_gcp.ocsf.json b/unittests/scans/prowler/example_output_gcp.ocsf.json new file mode 100644 index 00000000000..3267abff4b6 --- /dev/null +++ b/unittests/scans/prowler/example_output_gcp.ocsf.json @@ -0,0 +1,373 @@ +[ + { + "message": "Project does not have active API Keys.", + "metadata": { + "event_code": "apikeys_key_exists", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "5.4.0" + }, + "profiles": [ + "cloud", + "datetime" + ], + "tenant_uid": "", + "version": "1.4.0" + }, + "severity_id": 3, + "severity": "Medium", + "status": "New", + "status_code": "FAIL", + "status_detail": "Project does not have active API Keys.", + "status_id": 1, + "unmapped": { + "related_url": "", + "categories": [], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "", + "compliance": { + "MITRE-ATTACK": [ + "T1098" + ], + "CIS-2.0": [ + "1.12" + ], + "ENS-RD2022": [ + "op.acc.2.gcp.rbak.1" + ], + "CIS-3.0": [ + "1.12" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539640, + "created_time_dt": "2025-02-14T14:27:20.697446", + "desc": "API Keys should only be used for services in cases where other authentication methods are unavailable. Unused keys with their permissions in tact may still exist within a project. Keys are insecure because they can be viewed publicly, such as from within a browser, or they can be accessed on a device where the key resides. It is recommended to use standard authentication flow instead.", + "product_uid": "prowler", + "title": "Ensure API Keys Only Exist for Active Services", + "types": [], + "uid": "" + }, + "resources": [ + { + "region": "global", + "data": { + "details": "", + "metadata": { + "number": "", + "id": "", + "name": "", + "organization": { + "id": "", + "name": "organizations/", + "display_name": "prowler.com" + }, + "labels": { + "tag": "test", + "tag2": "test2", + "generative-language": "enabled" + }, + "lifecycle_state": "ACTIVE" + } + }, + "group": { + "name": "apikeys" + }, + "labels": [], + "name": "", + "type": "API Key", + "uid": "" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "cloud": { + "account": { + "name": "", + "type": "GCP Account", + "type_id": 5, + "uid": "", + "labels": [ + "tag:test" + ] + }, + "org": { + "name": "prowler.com", + "uid": "" + }, + "provider": "gcp", + "region": "global" + }, + "remediation": { + "desc": "To avoid the security risk in using API keys, it is recommended to use standard authentication flow instead.", + "references": [ + "gcloud alpha services api-keys delete", + "https://cloud.google.com/docs/authentication/api-keys" + ] + }, + "risk_details": "Security risks involved in using API-Keys appear below: API keys are simple encrypted strings, API keys do not identify the user or the application making the API request, API keys are typically accessible to clients, making it easy to discover and steal an API key.", + "time": 1739539640, + "time_dt": "2025-02-14T14:27:20.697446", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + }, + { + "message": "AR Container Analysis is not enabled in project .", + "metadata": { + "event_code": "artifacts_container_analysis_enabled", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "5.4.0" + }, + "profiles": [ + "cloud", + "datetime" + ], + "tenant_uid": "", + "version": "1.4.0" + }, + "severity_id": 3, + "severity": "Medium", + "status": "New", + "status_code": "FAIL", + "status_detail": "AR Container Analysis is not enabled in project .", + "status_id": 1, + "unmapped": { + "related_url": "https://cloud.google.com/artifact-analysis/docs", + "categories": [], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "By default, AR Container Analysis is disabled.", + "compliance": { + "MITRE-ATTACK": [ + "T1525" + ], + "ENS-RD2022": [ + "op.exp.4.r4.gcp.log.1", + "op.mon.3.gcp.scc.1" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539640, + "created_time_dt": "2025-02-14T14:27:20.697446", + "desc": "Scan images stored in Google Container Registry (GCR) for vulnerabilities using AR Container Analysis or a third-party provider. This helps identify and mitigate security risks associated with known vulnerabilities in container images.", + "product_uid": "prowler", + "title": "Ensure Image Vulnerability Analysis using AR Container Analysis or a third-party provider", + "types": [ + "Security", + "Configuration" + ], + "uid": "" + }, + "resources": [ + { + "region": "global", + "data": { + "details": "", + "metadata": { + "number": "538174383574", + "id": "", + "name": "", + "organization": { + "id": "", + "name": "organizations/", + "display_name": "prowler.com" + }, + "labels": { + "tag": "test", + "tag2": "test2", + "generative-language": "enabled" + }, + "lifecycle_state": "ACTIVE" + } + }, + "group": { + "name": "artifacts" + }, + "labels": [], + "name": "AR Container Analysis", + "type": "Service", + "uid": "containeranalysis.googleapis.com" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "cloud": { + "account": { + "name": "", + "type": "GCP Account", + "type_id": 5, + "uid": "", + "labels": [ + "tag:test" + ] + }, + "org": { + "name": "prowler.com", + "uid": "" + }, + "provider": "gcp", + "region": "global" + }, + "remediation": { + "desc": "Enable vulnerability scanning for images stored in Artifact Registry using AR Container Analysis or a third-party provider.", + "references": [ + "gcloud services enable containeranalysis.googleapis.com", + "https://cloud.google.com/artifact-analysis/docs/container-scanning-overview" + ] + }, + "risk_details": "Without image vulnerability scanning, container images stored in Artifact Registry may contain known vulnerabilities, increasing the risk of exploitation by malicious actors.", + "time": 1739539640, + "time_dt": "2025-02-14T14:27:20.697446", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + }, + { + "message": "Firewall does exposes port 3389 (RDP) to the internet.", + "metadata": { + "event_code": "compute_firewall_rdp_access_from_the_internet_allowed", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "5.4.0" + }, + "profiles": [ + "cloud", + "datetime" + ], + "tenant_uid": "", + "version": "1.4.0" + }, + "severity_id": 5, + "severity": "Critical", + "status": "New", + "status_code": "FAIL", + "status_detail": "Firewall does exposes port 3389 (RDP) to the internet.", + "status_id": 1, + "unmapped": { + "related_url": "", + "categories": [ + "internet-exposed" + ], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "", + "compliance": { + "MITRE-ATTACK": [ + "T1190", + "T1199", + "T1048", + "T1498", + "T1046" + ], + "CIS-2.0": [ + "3.7" + ], + "ENS-RD2022": [ + "mp.com.1.gcp.fw.1" + ], + "CIS-3.0": [ + "3.7" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539640, + "created_time_dt": "2025-02-14T14:27:20.697446", + "desc": "GCP `Firewall Rules` are specific to a `VPC Network`. Each rule either `allows` or `denies` traffic when its conditions are met. Its conditions allow users to specify the type of traffic, such as ports and protocols, and the source or destination of the traffic, including IP addresses, subnets, and instances. Firewall rules are defined at the VPC network level and are specific to the network in which they are defined. The rules themselves cannot be shared among networks. Firewall rules only support IPv4 traffic. When specifying a source for an ingress rule or a destination for an egress rule by address, an `IPv4` address or `IPv4 block in CIDR` notation can be used. Generic `(0.0.0.0/0)` incoming traffic from the Internet to a VPC or VM instance using `RDP` on `Port 3389` can be avoided.", + "product_uid": "prowler", + "title": "Ensure That RDP Access Is Restricted From the Internet", + "types": [], + "uid": "" + }, + "resources": [ + { + "region": "global", + "data": { + "details": "", + "metadata": { + "name": "", + "id": "", + "source_ranges": [ + "" + ], + "direction": "INGRESS", + "allowed_rules": [ + { + "IPProtocol": "tcp", + "ports": [ + "3389" + ] + } + ], + "project_id": "" + } + }, + "group": { + "name": "networking" + }, + "labels": [], + "name": "", + "type": "FirewallRule", + "uid": "" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "cloud": { + "account": { + "name": "", + "type": "GCP Account", + "type_id": 5, + "uid": "", + "labels": [ + "tag:test", + "tag2:test2" + ] + }, + "org": { + "name": "prowler.com", + "uid": "" + }, + "provider": "gcp", + "region": "global" + }, + "remediation": { + "desc": "Ensure that Google Cloud Virtual Private Cloud (VPC) firewall rules do not allow unrestricted access (i.e. 0.0.0.0/0) on TCP port 3389 in order to restrict Remote Desktop Protocol (RDP) traffic to trusted IP addresses or IP ranges only and reduce the attack surface. TCP port 3389 is used for secure remote GUI login to Windows VM instances by connecting a RDP client application with an RDP server.", + "references": [ + "https://docs.prowler.com/checks/gcp/google-cloud-networking-policies/bc_gcp_networking_2#terraform", + "https://docs.prowler.com/checks/gcp/google-cloud-networking-policies/bc_gcp_networking_2#cli-command", + "https://www.trendmicro.com/cloudoneconformity/knowledge-base/gcp/CloudVPC/unrestricted-rdp-access.html", + "https://cloud.google.com/vpc/docs/using-firewalls" + ] + }, + "risk_details": "Allowing unrestricted Remote Desktop Protocol (RDP) access can increase opportunities for malicious activities such as hacking, Man-In-The-Middle attacks (MITM) and Pass-The-Hash (PTH) attacks.", + "time": 1739539640, + "time_dt": "2025-02-14T14:27:20.697446", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + } +] diff --git a/unittests/scans/prowler/example_output_kubernetes.csv b/unittests/scans/prowler/example_output_kubernetes.csv new file mode 100644 index 00000000000..57b532fbda5 --- /dev/null +++ b/unittests/scans/prowler/example_output_kubernetes.csv @@ -0,0 +1,5 @@ +AUTH_METHOD;TIMESTAMP;ACCOUNT_UID;ACCOUNT_NAME;ACCOUNT_EMAIL;ACCOUNT_ORGANIZATION_UID;ACCOUNT_ORGANIZATION_NAME;ACCOUNT_TAGS;FINDING_UID;PROVIDER;CHECK_ID;CHECK_TITLE;CHECK_TYPE;STATUS;STATUS_EXTENDED;MUTED;SERVICE_NAME;SUBSERVICE_NAME;SEVERITY;RESOURCE_TYPE;RESOURCE_UID;RESOURCE_NAME;RESOURCE_DETAILS;RESOURCE_TAGS;PARTITION;REGION;DESCRIPTION;RISK;RELATED_URL;REMEDIATION_RECOMMENDATION_TEXT;REMEDIATION_RECOMMENDATION_URL;REMEDIATION_CODE_NATIVEIAC;REMEDIATION_CODE_TERRAFORM;REMEDIATION_CODE_CLI;REMEDIATION_CODE_OTHER;COMPLIANCE;CATEGORIES;DEPENDS_ON;RELATED_TO;NOTES;PROWLER_VERSION;ADDITIONAL_URLS +;2025-02-14 14:27:38.533897;;context: ;;;;;;kubernetes;apiserver_always_pull_images_plugin;Ensure that the admission control plugin AlwaysPullImages is set;;FAIL;AlwaysPullImages admission control plugin is not set in pod ;False;apiserver;;medium;KubernetesAPIServer;;;;;;namespace: kube-system;This check verifies that the AlwaysPullImages admission control plugin is enabled in the Kubernetes API server. This plugin ensures that every new pod always pulls the required images, enforcing image access control and preventing the use of possibly outdated or altered images.;Without AlwaysPullImages, once an image is pulled to a node, any pod can use it without any authorization check, potentially leading to security risks.;https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#alwayspullimages;Configure the API server to use the AlwaysPullImages admission control plugin to ensure image security and integrity.;https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers;https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-admission-control-plugin-alwayspullimages-is-set#kubernetes;;--enable-admission-plugins=...,AlwaysPullImages,...;;CIS-1.10: 1.2.11 | CIS-1.8: 1.2.11;cluster-security;;;Enabling AlwaysPullImages can increase network and registry load and decrease container startup speed. It may not be suitable for all environments.;;https://kubernetes.io/docs/concepts/containers/images/ | https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/ +;2025-02-14 14:27:38.533897;;context: ;;;;;;kubernetes;apiserver_anonymous_requests;Ensure that the --anonymous-auth argument is set to false;;FAIL;API Server does not have anonymous-auth enabled in pod ;False;apiserver;;high;KubernetesAPIServer;;;;;;namespace: kube-system;Disable anonymous requests to the API server. When enabled, requests that are not rejected by other configured authentication methods are treated as anonymous requests, which are then served by the API server. Disallowing anonymous requests strengthens security by ensuring all access is authenticated.;Enabling anonymous access to the API server can expose the cluster to unauthorized access and potential security vulnerabilities.;https://kubernetes.io/docs/admin/authentication/#anonymous-requests;Ensure the --anonymous-auth argument in the API server is set to false. This will reject all anonymous requests, enforcing authenticated access to the server.;https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/;https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-anonymous-auth-argument-is-set-to-false-1#kubernetes;;--anonymous-auth=false;;CIS-1.10: 1.2.1 | CIS-1.8: 1.2.1;trustboundaries;;;While anonymous access can be useful for health checks and discovery, consider the security implications for your specific environment.;;https://kubernetes.io/docs/concepts/containers/images/ | https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/ +;2025-02-14 14:27:38.533897;;context: ;;;;;;kubernetes;apiserver_audit_log_maxage_set;Ensure that the --audit-log-maxage argument is set to 30 or as appropriate;;FAIL;Audit log max age is not set to 30 or as appropriate in pod ;False;apiserver;;medium;KubernetesAPIServer;;;;;;namespace: kube-system;This check ensures that the Kubernetes API server is configured with an appropriate audit log retention period. Setting --audit-log-maxage to 30 or as per business requirements helps in maintaining logs for sufficient time to investigate past events.;Without an adequate log retention period, there may be insufficient audit history to investigate and analyze past events or security incidents.;https://kubernetes.io/docs/concepts/cluster-administration/audit/;Configure the API server audit log retention period to retain logs for at least 30 days or as per your organization's requirements.;https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/;https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-audit-log-maxage-argument-is-set-to-30-or-as-appropriate#kubernetes;;--audit-log-maxage=30;;CIS-1.10: 1.2.17 | CIS-1.8: 1.2.18;logging;;;Ensure the audit log retention period is set appropriately to balance between storage constraints and the need for historical data.;;https://kubernetes.io/docs/concepts/containers/images/ | https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/ +;2025-02-14 14:27:38.533897;;context: ;;;;;;kubernetes;apiserver_audit_log_maxbackup_set;Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate;;FAIL;Audit log max backup is not set to 10 or as appropriate in pod ;False;apiserver;;medium;KubernetesAPIServer;;;;;;namespace: kube-system;This check ensures that the Kubernetes API server is configured with an appropriate number of audit log backups. Setting --audit-log-maxbackup to 10 or as per business requirements helps maintain a sufficient log backup for investigations or analysis.;Without an adequate number of audit log backups, there may be insufficient log history to investigate past events or security incidents.;https://kubernetes.io/docs/concepts/cluster-administration/audit/;Configure the API server audit log backup retention to 10 or as per your organization's requirements.;https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/;https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate#kubernetes;;--audit-log-maxbackup=10;;CIS-1.10: 1.2.18 | CIS-1.8: 1.2.19;logging;;;Ensure the audit log backup retention period is set appropriately to balance between storage constraints and the need for historical data.;;https://kubernetes.io/docs/concepts/containers/images/ | https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/ diff --git a/unittests/scans/prowler/example_output_kubernetes.ocsf.json b/unittests/scans/prowler/example_output_kubernetes.ocsf.json new file mode 100644 index 00000000000..6e7dffb48a2 --- /dev/null +++ b/unittests/scans/prowler/example_output_kubernetes.ocsf.json @@ -0,0 +1,538 @@ +[ + { + "message": "AlwaysPullImages admission control plugin is not set in pod .", + "metadata": { + "event_code": "apiserver_always_pull_images_plugin", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "5.4.0" + }, + "profiles": [ + "container", + "datetime" + ], + "version": "1.4.0" + }, + "severity_id": 3, + "severity": "Medium", + "status": "New", + "status_code": "FAIL", + "status_detail": "AlwaysPullImages admission control plugin is not set in pod .", + "status_id": 1, + "unmapped": { + "related_url": "https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#alwayspullimages", + "categories": [ + "cluster-security" + ], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "Enabling AlwaysPullImages can increase network and registry load and decrease container startup speed. It may not be suitable for all environments.", + "compliance": { + "CIS-1.10": [ + "1.2.11" + ], + "CIS-1.8": [ + "1.2.11" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539658, + "created_time_dt": "2025-02-14T14:27:38.533897", + "desc": "This check verifies that the AlwaysPullImages admission control plugin is enabled in the Kubernetes API server. This plugin ensures that every new pod always pulls the required images, enforcing image access control and preventing the use of possibly outdated or altered images.", + "product_uid": "prowler", + "title": "Ensure that the admission control plugin AlwaysPullImages is set", + "types": [], + "uid": "" + }, + "resources": [ + { + "data": { + "details": "", + "metadata": { + "name": "", + "uid": "", + "namespace": "", + "labels": { + "component": "kube-apiserver", + "tier": "control-plane" + }, + "annotations": { + "kubernetes.io/config.source": "file" + }, + "node_name": "", + "service_account": null, + "status_phase": "Running", + "pod_ip": "", + "host_ip": "", + "host_pid": null, + "host_ipc": null, + "host_network": "True", + "security_context": { + "app_armor_profile": null, + "fs_group": null, + "fs_group_change_policy": null, + "run_as_group": null, + "run_as_non_root": null, + "run_as_user": null, + "se_linux_change_policy": null, + "se_linux_options": null, + "seccomp_profile": { + "localhost_profile": null, + "type": "RuntimeDefault" + }, + "supplemental_groups": null, + "supplemental_groups_policy": null, + "sysctls": null, + "windows_options": null + }, + "containers": { + "kube-apiserver": { + "name": "kube-apiserver", + "image": "", + "command": [ + "" + ], + "ports": null, + "env": null, + "security_context": {} + } + } + } + }, + "group": { + "name": "apiserver" + }, + "labels": [], + "name": "", + "namespace": "", + "type": "KubernetesAPIServer", + "uid": "" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "remediation": { + "desc": "Configure the API server to use the AlwaysPullImages admission control plugin to ensure image security and integrity.", + "references": [ + "https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-admission-control-plugin-alwayspullimages-is-set#kubernetes", + "--enable-admission-plugins=...,AlwaysPullImages,...", + "https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers" + ] + }, + "risk_details": "Without AlwaysPullImages, once an image is pulled to a node, any pod can use it without any authorization check, potentially leading to security risks.", + "time": 1739539658, + "time_dt": "2025-02-14T14:27:38.533897", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + }, + { + "message": "Audit log max age is not set to 30 or as appropriate in pod .", + "metadata": { + "event_code": "apiserver_audit_log_maxage_set", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "5.4.0" + }, + "profiles": [ + "container", + "datetime" + ], + "version": "1.4.0" + }, + "severity_id": 3, + "severity": "Medium", + "status": "New", + "status_code": "FAIL", + "status_detail": "Audit log max age is not set to 30 or as appropriate in pod .", + "status_id": 1, + "unmapped": { + "related_url": "https://kubernetes.io/docs/concepts/cluster-administration/audit/", + "categories": [ + "logging" + ], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "Ensure the audit log retention period is set appropriately to balance between storage constraints and the need for historical data.", + "compliance": { + "CIS-1.10": [ + "1.2.17" + ], + "CIS-1.8": [ + "1.2.18" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539658, + "created_time_dt": "2025-02-14T14:27:38.533897", + "desc": "This check ensures that the Kubernetes API server is configured with an appropriate audit log retention period. Setting --audit-log-maxage to 30 or as per business requirements helps in maintaining logs for sufficient time to investigate past events.", + "product_uid": "prowler", + "title": "Ensure that the --audit-log-maxage argument is set to 30 or as appropriate", + "types": [], + "uid": "" + }, + "resources": [ + { + "data": { + "details": "", + "metadata": { + "name": "", + "uid": "", + "namespace": "", + "labels": { + "component": "kube-apiserver", + "tier": "control-plane" + }, + "annotations": { + "kubernetes.io/config.source": "file" + }, + "node_name": "", + "service_account": null, + "status_phase": "Running", + "pod_ip": "", + "host_ip": "", + "host_pid": null, + "host_ipc": null, + "host_network": "True", + "security_context": { + "app_armor_profile": null, + "fs_group": null, + "fs_group_change_policy": null, + "run_as_group": null, + "run_as_non_root": null, + "run_as_user": null, + "se_linux_change_policy": null, + "se_linux_options": null, + "seccomp_profile": { + "localhost_profile": null, + "type": "RuntimeDefault" + }, + "supplemental_groups": null, + "supplemental_groups_policy": null, + "sysctls": null, + "windows_options": null + }, + "containers": { + "kube-apiserver": { + "name": "kube-apiserver", + "image": "", + "command": [ + "" + ], + "ports": null, + "env": null, + "security_context": {} + } + } + } + }, + "group": { + "name": "apiserver" + }, + "labels": [], + "name": "", + "namespace": "", + "type": "KubernetesAPIServer", + "uid": "" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "remediation": { + "desc": "Configure the API server audit log retention period to retain logs for at least 30 days or as per your organization's requirements.", + "references": [ + "https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-audit-log-maxage-argument-is-set-to-30-or-as-appropriate#kubernetes", + "--audit-log-maxage=30", + "https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/" + ] + }, + "risk_details": "Without an adequate log retention period, there may be insufficient audit history to investigate and analyze past events or security incidents.", + "time": 1739539658, + "time_dt": "2025-02-14T14:27:38.533897", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + }, + { + "message": "Audit log max backup is not set to 10 or as appropriate in pod .", + "metadata": { + "event_code": "apiserver_audit_log_maxbackup_set", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "5.4.0" + }, + "profiles": [ + "container", + "datetime" + ], + "version": "1.4.0" + }, + "severity_id": 3, + "severity": "Medium", + "status": "New", + "status_code": "FAIL", + "status_detail": "Audit log max backup is not set to 10 or as appropriate in pod .", + "status_id": 1, + "unmapped": { + "related_url": "https://kubernetes.io/docs/concepts/cluster-administration/audit/", + "categories": [ + "logging" + ], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "Ensure the audit log backup retention period is set appropriately to balance between storage constraints and the need for historical data.", + "compliance": { + "CIS-1.10": [ + "1.2.18" + ], + "CIS-1.8": [ + "1.2.19" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539658, + "created_time_dt": "2025-02-14T14:27:38.533897", + "desc": "This check ensures that the Kubernetes API server is configured with an appropriate number of audit log backups. Setting --audit-log-maxbackup to 10 or as per business requirements helps maintain a sufficient log backup for investigations or analysis.", + "product_uid": "prowler", + "title": "Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate", + "types": [], + "uid": "" + }, + "resources": [ + { + "data": { + "details": "", + "metadata": { + "name": "", + "uid": "", + "namespace": "", + "labels": { + "component": "kube-apiserver", + "tier": "control-plane" + }, + "annotations": { + "kubernetes.io/config.source": "file" + }, + "node_name": "", + "service_account": null, + "status_phase": "Running", + "pod_ip": "", + "host_ip": "", + "host_pid": null, + "host_ipc": null, + "host_network": "True", + "security_context": { + "app_armor_profile": null, + "fs_group": null, + "fs_group_change_policy": null, + "run_as_group": null, + "run_as_non_root": null, + "run_as_user": null, + "se_linux_change_policy": null, + "se_linux_options": null, + "seccomp_profile": { + "localhost_profile": null, + "type": "RuntimeDefault" + }, + "supplemental_groups": null, + "supplemental_groups_policy": null, + "sysctls": null, + "windows_options": null + }, + "containers": { + "kube-apiserver": { + "name": "kube-apiserver", + "image": "", + "command": [ + "" + ], + "ports": null, + "env": null, + "security_context": {} + } + } + } + }, + "group": { + "name": "apiserver" + }, + "labels": [], + "name": "", + "namespace": "", + "type": "KubernetesAPIServer", + "uid": "" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "remediation": { + "desc": "Configure the API server audit log backup retention to 10 or as per your organization's requirements.", + "references": [ + "https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate#kubernetes", + "--audit-log-maxbackup=10", + "https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/" + ] + }, + "risk_details": "Without an adequate number of audit log backups, there may be insufficient log history to investigate past events or security incidents.", + "time": 1739539658, + "time_dt": "2025-02-14T14:27:38.533897", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + }, + { + "message": "Audit log max backup is not set to 10 or as appropriate in pod .", + "metadata": { + "event_code": "apiserver_audit_log_maxbackup_set", + "product": { + "name": "Prowler", + "uid": "prowler", + "vendor_name": "Prowler", + "version": "5.4.0" + }, + "profiles": [ + "container", + "datetime" + ], + "version": "1.4.0" + }, + "severity_id": 3, + "severity": "Medium", + "status": "New", + "status_code": "FAIL", + "status_detail": "Audit log max backup is not set to 10 or as appropriate in pod .", + "status_id": 1, + "unmapped": { + "related_url": "https://kubernetes.io/docs/concepts/cluster-administration/audit/", + "categories": [ + "logging" + ], + "depends_on": [], + "related_to": [], + "additional_urls": [], + "notes": "Ensure the audit log backup retention period is set appropriately to balance between storage constraints and the need for historical data.", + "compliance": { + "CIS-1.10": [ + "1.2.18" + ], + "CIS-1.8": [ + "1.2.19" + ] + } + }, + "activity_name": "Create", + "activity_id": 1, + "finding_info": { + "created_time": 1739539658, + "created_time_dt": "2025-02-14T14:27:38.533897", + "desc": "This check ensures that the Kubernetes API server is configured with an appropriate number of audit log backups. Setting --audit-log-maxbackup to 10 or as per business requirements helps maintain a sufficient log backup for investigations or analysis.", + "product_uid": "prowler", + "title": "Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate", + "types": [], + "uid": "" + }, + "resources": [ + { + "data": { + "details": "", + "metadata": { + "name": "", + "uid": "", + "namespace": "", + "labels": { + "component": "kube-apiserver", + "tier": "control-plane" + }, + "annotations": { + "kubernetes.io/config.source": "file" + }, + "node_name": "", + "service_account": null, + "status_phase": "Running", + "pod_ip": "", + "host_ip": "", + "host_pid": null, + "host_ipc": null, + "host_network": "True", + "security_context": { + "app_armor_profile": null, + "fs_group": null, + "fs_group_change_policy": null, + "run_as_group": null, + "run_as_non_root": null, + "run_as_user": null, + "se_linux_change_policy": null, + "se_linux_options": null, + "seccomp_profile": { + "localhost_profile": null, + "type": "RuntimeDefault" + }, + "supplemental_groups": null, + "supplemental_groups_policy": null, + "sysctls": null, + "windows_options": null + }, + "containers": { + "kube-apiserver": { + "name": "kube-apiserver", + "image": "", + "command": [ + "" + ], + "ports": null, + "env": null, + "security_context": {} + } + } + } + }, + "group": { + "name": "apiserver" + }, + "labels": [], + "name": "", + "namespace": "", + "type": "KubernetesAPIServer", + "uid": "" + } + ], + "category_name": "Findings", + "category_uid": 2, + "class_name": "Detection Finding", + "class_uid": 2004, + "remediation": { + "desc": "Configure the API server audit log backup retention to 10 or as per your organization's requirements.", + "references": [ + "https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate#kubernetes", + "--audit-log-maxbackup=10", + "https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/" + ] + }, + "risk_details": "Without an adequate number of audit log backups, there may be insufficient log history to investigate past events or security incidents.", + "time": 1739539658, + "time_dt": "2025-02-14T14:27:38.533897", + "type_uid": 200401, + "type_name": "Detection Finding: Create" + } +] diff --git a/unittests/scans/prowler/prowler_zero_vul.csv b/unittests/scans/prowler/prowler_zero_vul.csv new file mode 100644 index 00000000000..04a160e85fe --- /dev/null +++ b/unittests/scans/prowler/prowler_zero_vul.csv @@ -0,0 +1 @@ +AUTH_METHOD;TIMESTAMP;ACCOUNT_UID;ACCOUNT_NAME;ACCOUNT_EMAIL;ACCOUNT_ORGANIZATION_UID;ACCOUNT_ORGANIZATION_NAME;ACCOUNT_TAGS;FINDING_UID;PROVIDER;CHECK_ID;CHECK_TITLE;CHECK_TYPE;STATUS;STATUS_EXTENDED;MUTED;SERVICE_NAME;SUBSERVICE_NAME;SEVERITY;RESOURCE_TYPE;RESOURCE_UID;RESOURCE_NAME;RESOURCE_DETAILS;RESOURCE_TAGS;PARTITION;REGION;DESCRIPTION;RISK;RELATED_URL;REMEDIATION_RECOMMENDATION_TEXT;REMEDIATION_RECOMMENDATION_URL;REMEDIATION_CODE_NATIVEIAC;REMEDIATION_CODE_TERRAFORM;REMEDIATION_CODE_CLI;REMEDIATION_CODE_OTHER;COMPLIANCE;CATEGORIES;DEPENDS_ON;RELATED_TO;NOTES;PROWLER_VERSION;ADDITIONAL_URLS diff --git a/unittests/scans/prowler/prowler_zero_vul.json b/unittests/scans/prowler/prowler_zero_vul.json new file mode 100644 index 00000000000..c9c513b9b8c --- /dev/null +++ b/unittests/scans/prowler/prowler_zero_vul.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/unittests/test_apiv2_user.py b/unittests/test_apiv2_user.py index d781a4ea384..6a5b1998659 100644 --- a/unittests/test_apiv2_user.py +++ b/unittests/test_apiv2_user.py @@ -1,7 +1,10 @@ from django.urls import reverse +from django.utils import timezone from rest_framework.authtoken.models import Token from rest_framework.test import APIClient, APITestCase +from dojo.models import Global_Role, Role, User, UserContactInfo + class UserTest(APITestCase): @@ -126,3 +129,112 @@ def test_user_deactivate(self): "password": password, }, format="json") self.assertEqual(r.status_code, 400, r.content[:1000]) + + def test_user_reset_api_token_as_superuser(self): + r = self.client.post(reverse("user-list"), { + "username": "api-user-reset-1", + "email": "admin@dojo.com", + "password": "testTEST1234!@#$", + }, format="json") + self.assertEqual(r.status_code, 201, r.content[:1000]) + user_id = r.json()["id"] + + target_user = User.objects.get(id=user_id) + # Tokens aren't created automatically for new users; ensure one exists. + old_token = Token.objects.get_or_create(user=target_user)[0].key + + url = "{}{}/reset_api_token/".format(reverse("user-list"), user_id) + r = self.client.post(url) + self.assertEqual(r.status_code, 204, r.content[:1000]) + + new_token = Token.objects.get(user=target_user).key + self.assertNotEqual(old_token, new_token) + + uci = UserContactInfo.objects.get(user=target_user) + self.assertIsNotNone(uci.token_last_reset) + self.assertLess(abs((timezone.now() - uci.token_last_reset).total_seconds()), 30) + + def test_user_reset_api_token_denies_self(self): + admin_user = User.objects.get(username="admin") + admin_token_before = Token.objects.get(user=admin_user).key + + url = "{}{}/reset_api_token/".format(reverse("user-list"), admin_user.id) + r = self.client.post(url) + self.assertIn(r.status_code, {400, 403}, r.content[:1000]) + + admin_token_after = Token.objects.get(user=admin_user).key + self.assertEqual(admin_token_before, admin_token_after) + + def test_user_reset_api_token_denies_non_privileged(self): + # Create a target + r = self.client.post(reverse("user-list"), { + "username": "api-user-reset-target", + "email": "admin@dojo.com", + "password": "testTEST1234!@#$", + }, format="json") + self.assertEqual(r.status_code, 201, r.content[:1000]) + target_id = r.json()["id"] + + # Create a non-privileged user and authenticate as them + password = "testTEST1234!@#$" + r = self.client.post(reverse("user-list"), { + "username": "api-user-nonpriv", + "email": "admin@dojo.com", + "password": password, + }, format="json") + self.assertEqual(r.status_code, 201, r.content[:1000]) + + token_resp = self.client.post(reverse("api-token-auth"), { + "username": "api-user-nonpriv", + "password": password, + }, format="json") + self.assertEqual(token_resp.status_code, 200, token_resp.content[:1000]) + nonpriv_token = token_resp.json()["token"] + + nonpriv_client = APIClient() + nonpriv_client.credentials(HTTP_AUTHORIZATION="Token " + nonpriv_token) + + url = "{}{}/reset_api_token/".format(reverse("user-list"), target_id) + r = nonpriv_client.post(url) + self.assertEqual(r.status_code, 403, r.content[:1000]) + + def test_user_reset_api_token_allows_global_owner(self): + # Create a global-owner user (not superuser) + password = "testTEST1234!@#$" + r = self.client.post(reverse("user-list"), { + "username": "api-user-global-owner", + "email": "admin@dojo.com", + "password": password, + }, format="json") + self.assertEqual(r.status_code, 201, r.content[:1000]) + global_owner = User.objects.get(username="api-user-global-owner") + + owner_role, _ = Role.objects.get_or_create(name="Owner", defaults={"is_owner": True}) + if not owner_role.is_owner: + owner_role.is_owner = True + owner_role.save(update_fields=["is_owner"]) + Global_Role.objects.update_or_create(user=global_owner, defaults={"role": owner_role}) + + # Create target + r = self.client.post(reverse("user-list"), { + "username": "api-user-reset-2", + "email": "admin@dojo.com", + "password": "testTEST1234!@#$", + }, format="json") + self.assertEqual(r.status_code, 201, r.content[:1000]) + target_id = r.json()["id"] + + # Authenticate as global owner + token_resp = self.client.post(reverse("api-token-auth"), { + "username": "api-user-global-owner", + "password": password, + }, format="json") + self.assertEqual(token_resp.status_code, 200, token_resp.content[:1000]) + go_token = token_resp.json()["token"] + + go_client = APIClient() + go_client.credentials(HTTP_AUTHORIZATION="Token " + go_token) + + url = "{}{}/reset_api_token/".format(reverse("user-list"), target_id) + r = go_client.post(url) + self.assertEqual(r.status_code, 204, r.content[:1000]) diff --git a/unittests/test_apply_finding_template.py b/unittests/test_apply_finding_template.py index e845a0752bb..685d39fcfe9 100644 --- a/unittests/test_apply_finding_template.py +++ b/unittests/test_apply_finding_template.py @@ -1,14 +1,30 @@ import datetime -from unittest import skip +from crum import impersonate from django.contrib.auth.models import User from django.contrib.messages.storage.fallback import FallbackStorage +from django.core.exceptions import PermissionDenied from django.http import Http404 from django.test.client import RequestFactory from django.utils import timezone from dojo.finding import views -from dojo.models import Engagement, Finding, Finding_Template, Product, Product_Type, System_Settings, Test, Test_Type +from dojo.finding.helper import save_endpoints_template, save_vulnerability_ids_template +from dojo.models import ( + Engagement, + Finding, + Finding_Template, + Notes, + Product, + Product_Member, + Product_Type, + Role, + System_Settings, + Test, + Test_Type, + Vulnerability_Id, +) +from dojo.test import views as test_views from .dojo_test_case import DojoTestCase @@ -21,8 +37,8 @@ def create(): settings.save() p = Product() - p.Name = "Test Product" - p.Description = "Product for Testing Apply Template functionality" + p.name = "Test Product" + p.description = "Product for Testing Apply Template functionality" p.prod_type = Product_Type.objects.get(id=1) p.save() @@ -53,7 +69,9 @@ def create(): f.reporter = user f.last_reviewed = timezone.now() f.last_reviewed_by = user + f.cve = None # Set explicitly as it's required (blank=False) f.save() + return f class FindingTemplateMother: @@ -67,6 +85,7 @@ def create(): tmp.mitigation = "Finding Template Mitigation" tmp.impact = "Finding Template Impact" tmp.save() + return tmp class FindingTemplateTestUtil: @@ -79,10 +98,24 @@ def create_user(is_staff): user_count = User.objects.count() user = User() user.is_staff = is_staff + user.is_superuser = is_staff # Superuser has all permissions user.username = "TestUser" + str(user_count) user.save() return user + @staticmethod + def create_user_with_role(product, role_name, *, is_staff=False): + """Create a user with a specific role on a product""" + user_count = User.objects.count() + user = User() + user.is_staff = is_staff + user.is_superuser = False + user.username = f"TestUser{role_name}{user_count}" + user.save() + role = Role.objects.get(name=role_name) + Product_Member(user=user, product=product, role=role).save() + return user + @staticmethod def create_get_request(user, path): rf = RequestFactory() @@ -104,15 +137,13 @@ def create_post_request(user, path, data): return post_request -@skip("outdated so doesn't work with current fixture") class TestApplyFindingTemplate(DojoTestCase): fixtures = ["dojo_testdata.json"] - apply_template_url = "finding/2/2/apply_template_to_finding" - def setUp(self): - FindingMother.create() - FindingTemplateMother.create() + self.finding = FindingMother.create() + self.template = FindingTemplateMother.create() + self.apply_template_url = f"finding/{self.finding.id}/{self.template.id}/apply_template_to_finding" def make_request(self, user_is_staff, finding_id, template_id, data=None): user = FindingTemplateTestUtil.create_user(user_is_staff) @@ -122,20 +153,22 @@ def make_request(self, user_is_staff, finding_id, template_id, data=None): else: request = FindingTemplateTestUtil.create_get_request(user, self.apply_template_url) - return views.apply_template_to_finding(request, finding_id, template_id) + with impersonate(user): + return views.apply_template_to_finding(request, fid=finding_id, tid=template_id) def test_apply_template_to_finding_with_data_does_not_display_error_success(self): - result = self.make_request(user_is_staff=True, finding_id=1, template_id=1, + result = self.make_request(user_is_staff=True, finding_id=self.finding.id, template_id=self.template.id, data={"title": "Finding for Testing Apply Template functionality", "cwe": "89", "severity": "High", "description": "Finding for Testing Apply Template Functionality", "mitigation": "template mitigation", "impact": "template impact"}) - self.assertNotContains(result, "There appears to be errors on the form", 302) + self.assertEqual(result.status_code, 302) + self.assertNotIn("There appears to be errors on the form", str(result)) def test_apply_template_to_finding_with_data_returns_to_view_success(self): - result = self.make_request(user_is_staff=True, finding_id=1, template_id=1, + result = self.make_request(user_is_staff=True, finding_id=self.finding.id, template_id=self.template.id, data={"title": "Finding for Testing Apply Template functionality", "cwe": "89", "severity": "High", @@ -144,7 +177,7 @@ def test_apply_template_to_finding_with_data_returns_to_view_success(self): "impact": "template impact"}) self.assertIsNotNone(result) self.assertEqual(302, result.status_code) - self.assertEqual("/finding/1", result.url) + self.assertEqual(f"/finding/{self.finding.id}", result.url) def test_apply_template_to_finding_with_data_saves_success(self): test_title = "Finding for Testing Apply Template functionality" @@ -154,7 +187,7 @@ def test_apply_template_to_finding_with_data_saves_success(self): test_mitigation = "template mitigation" test_impact = "template impact" - self.make_request(user_is_staff=True, finding_id=1, template_id=1, + self.make_request(user_is_staff=True, finding_id=self.finding.id, template_id=self.template.id, data={"title": test_title, "cwe": test_cwe, "severity": test_severity, @@ -162,8 +195,9 @@ def test_apply_template_to_finding_with_data_saves_success(self): "mitigation": test_mitigation, "impact": test_impact}) - f = Finding.objects.get(id=1) - self.assertEqual(test_title, f.title) + f = Finding.objects.get(id=self.finding.id) + # Title is automatically title-cased by Finding.save() + self.assertEqual("Finding for Testing Apply Template Functionality", f.title) self.assertEqual(test_cwe, f.cwe) self.assertEqual(test_severity, f.severity) self.assertEqual(test_description, f.description) @@ -171,7 +205,9 @@ def test_apply_template_to_finding_with_data_saves_success(self): self.assertEqual(test_impact, f.impact) def test_unauthorized_apply_template_to_finding_fails(self): - result = self.make_request(user_is_staff=False, finding_id=1, template_id=1, + """Test that a non-superuser without permissions cannot apply template""" + with self.assertRaises(PermissionDenied): + self.make_request(user_is_staff=False, finding_id=self.finding.id, template_id=self.template.id, data={"title": "Finding for Testing Apply Template functionality", "cwe": "89", "severity": "High", @@ -179,23 +215,59 @@ def test_unauthorized_apply_template_to_finding_fails(self): "mitigation": "template mitigation", "impact": "template impact"}, ) - self.assertEqual(302, result.status_code) - self.assertIn("login", result.url) + + def test_reader_role_cannot_apply_template(self): + """Test that a Reader role user (read-only) cannot apply template""" + reader_user = FindingTemplateTestUtil.create_user_with_role( + self.finding.test.engagement.product, "Reader", is_staff=False, + ) + request = FindingTemplateTestUtil.create_post_request( + reader_user, self.apply_template_url, + data={"title": "Finding for Testing Apply Template functionality", + "cwe": "89", + "severity": "High", + "description": "Finding for Testing Apply Template Functionality", + "mitigation": "template mitigation", + "impact": "template impact"}, + ) + with impersonate(reader_user), self.assertRaises(PermissionDenied): + views.apply_template_to_finding(request, fid=self.finding.id, tid=self.template.id) + + def test_writer_role_can_apply_template(self): + """Test that a Writer role user (non-staff) can apply template""" + writer_user = FindingTemplateTestUtil.create_user_with_role( + self.finding.test.engagement.product, "Writer", is_staff=False, + ) + request = FindingTemplateTestUtil.create_post_request( + writer_user, self.apply_template_url, + data={"title": "Finding for Testing Apply Template functionality", + "cwe": "89", + "severity": "High", + "description": "Finding for Testing Apply Template Functionality", + "mitigation": "template mitigation", + "impact": "template impact"}, + ) + with impersonate(writer_user): + result = views.apply_template_to_finding(request, fid=self.finding.id, tid=self.template.id) + self.assertEqual(302, result.status_code) + self.assertEqual(f"/finding/{self.finding.id}", result.url) def test_apply_template_to_finding_with_illegal_finding_fails(self): - self.make_request(user_is_staff=True, finding_id=None, template_id=1) + with self.assertRaises(Http404): + self.make_request(user_is_staff=True, finding_id=99999, template_id=self.template.id) def test_apply_template_to_finding_with_illegal_template_fails(self): - self.make_request(user_is_staff=True, finding_id=1, template_id=None) + with self.assertRaises(Http404): + self.make_request(user_is_staff=True, finding_id=self.finding.id, template_id=99999) def test_apply_template_to_finding_with_no_data_returns_view_success(self): - result = self.make_request(user_is_staff=True, finding_id=1, template_id=1, data=None) + result = self.make_request(user_is_staff=True, finding_id=self.finding.id, template_id=self.template.id, data=None) self.assertIsNotNone(result) self.assertEqual(302, result.status_code) - self.assertEqual("/finding/1", result.url) + self.assertEqual(f"/finding/{self.finding.id}", result.url) def test_apply_template_to_finding_without_required_field_displays_field_title_success(self): - result = self.make_request(user_is_staff=True, finding_id=1, template_id=1, + result = self.make_request(user_is_staff=True, finding_id=self.finding.id, template_id=self.template.id, data={"title": "", "cwe": "89", "severity": "High", @@ -205,7 +277,7 @@ def test_apply_template_to_finding_without_required_field_displays_field_title_s self.assertContains(result, "The title is required.") def test_apply_template_to_finding_without_required_field_displays_error_success(self): - result = self.make_request(user_is_staff=True, finding_id=1, template_id=1, + result = self.make_request(user_is_staff=True, finding_id=self.finding.id, template_id=self.template.id, data={"title": "", "cwe": "89", "severity": "High", @@ -215,14 +287,13 @@ def test_apply_template_to_finding_without_required_field_displays_error_success self.assertContains(result, "There appears to be errors on the form") -@skip("outdated so doesn't work with current fixture") class TestFindTemplateToApply(DojoTestCase): fixtures = ["dojo_testdata.json"] - choose_template_url = "finding/2/find_template_to_apply" def setUp(self): - FindingMother.create() + self.finding = FindingMother.create() FindingTemplateMother.create() + self.choose_template_url = f"finding/{self.finding.id}/find_template_to_apply" def make_request(self, user_is_staff, finding_id, data=None): user = FindingTemplateTestUtil.create_user(user_is_staff) @@ -232,34 +303,33 @@ def make_request(self, user_is_staff, finding_id, data=None): else: request = FindingTemplateTestUtil.create_get_request(user, self.choose_template_url) - return views.find_template_to_apply(request, finding_id) + with impersonate(user): + return views.find_template_to_apply(request, fid=finding_id) def test_unauthorized_find_template_to_apply_fails(self): - result = self.make_request(user_is_staff=False, finding_id=1) - self.assertEqual(302, result.status_code) - self.assertIn("login", result.url) + with self.assertRaises(PermissionDenied): + self.make_request(user_is_staff=False, finding_id=self.finding.id) def test_authorized_find_template_to_apply_success(self): - result = self.make_request(user_is_staff=True, finding_id=1) + result = self.make_request(user_is_staff=True, finding_id=self.finding.id) self.assertEqual(200, result.status_code) def test_find_template_to_apply_displays_templates_success(self): - result = self.make_request(user_is_staff=True, finding_id=1) + result = self.make_request(user_is_staff=True, finding_id=self.finding.id) self.assertContains(result, "Finding Template for Testing Apply Template functionality") def test_find_template_to_apply_displays_breadcrumb(self): - result = self.make_request(user_is_staff=True, finding_id=1) + result = self.make_request(user_is_staff=True, finding_id=self.finding.id) self.assertContains(result, "Apply Template to Finding") -@skip("outdated so doesn't work with current fixture") class TestChooseFindingTemplateOptions(DojoTestCase): fixtures = ["dojo_testdata.json"] - finding_template_options_url = "finding/2/2/choose_finding_template_options" def setUp(self): - FindingMother.create() - FindingTemplateMother.create() + self.finding = FindingMother.create() + self.template = FindingTemplateMother.create() + self.finding_template_options_url = f"finding/{self.template.id}/{self.finding.id}/choose_finding_template_options" def make_request(self, user_is_staff, finding_id, template_id, data=None): user = FindingTemplateTestUtil.create_user(user_is_staff) @@ -269,27 +339,258 @@ def make_request(self, user_is_staff, finding_id, template_id, data=None): else: request = FindingTemplateTestUtil.create_get_request(user, self.finding_template_options_url) - return views.choose_finding_template_options(request, finding_id, template_id) + with impersonate(user): + return views.choose_finding_template_options(request, tid=template_id, fid=finding_id) def test_unauthorized_choose_finding_template_options_fails(self): - result = self.make_request(user_is_staff=False, finding_id=1, template_id=1) - self.assertEqual(302, result.status_code) - self.assertIn("login", result.url) + with self.assertRaises(PermissionDenied): + self.make_request(user_is_staff=False, finding_id=self.finding.id, template_id=self.template.id) def test_authorized_choose_finding_template_options_success(self): - result = self.make_request(user_is_staff=True, finding_id=1, template_id=1) + result = self.make_request(user_is_staff=True, finding_id=self.finding.id, template_id=self.template.id) self.assertEqual(200, result.status_code) def test_choose_finding_template_options_with_invalid_finding_fails(self): with self.assertRaises(Http404): - result = self.make_request(user_is_staff=True, finding_id=0, template_id=1) - self.assertEqual(404, result.status_code) + self.make_request(user_is_staff=True, finding_id=99999, template_id=self.template.id) def test_choose_finding_template_options_with_invalid_template_fails(self): with self.assertRaises(Http404): - result = self.make_request(user_is_staff=True, finding_id=1, template_id=0) - self.assertEqual(404, result.status_code) + self.make_request(user_is_staff=True, finding_id=self.finding.id, template_id=99999) def test_choose_finding_template_options_with_valid_finding_and_template_renders_apply_finding_template_view(self): - result = self.make_request(user_is_staff=True, finding_id=1, template_id=1) + result = self.make_request(user_is_staff=True, finding_id=self.finding.id, template_id=self.template.id) self.assertContains(result, "

    Apply template to a Finding

    ") + + +class TestMkTemplate(DojoTestCase): + fixtures = ["dojo_testdata.json"] + + def setUp(self): + self.finding = FindingMother.create() + self.user = FindingTemplateTestUtil.create_user(is_staff=True) + self.user.is_superuser = True + self.user.save() + + def make_request(self, user, finding_id): + rf = RequestFactory() + request = rf.get(f"/finding/{finding_id}/mktemplate") + request.user = user + request.session = {} + messages = FallbackStorage(request) + request._messages = messages + return views.mktemplate(request, finding_id) + + def test_mktemplate_creates_template_from_finding(self): + """Test that mktemplate creates a template from an existing finding""" + # Verify no template exists with this title + initial_count = Finding_Template.objects.filter(title=self.finding.title).count() + self.assertEqual(initial_count, 0) + + # Create template from finding + result = self.make_request(self.user, self.finding.id) + + # Verify redirect to finding view + self.assertEqual(result.status_code, 302) + self.assertEqual(result.url, f"/finding/{self.finding.id}") + + # Verify template was created + templates = Finding_Template.objects.filter(title=self.finding.title) + self.assertEqual(templates.count(), 1) + + template = templates.first() + self.assertEqual(template.title, self.finding.title) + self.assertEqual(template.cwe, self.finding.cwe) + self.assertEqual(template.severity, self.finding.severity) + self.assertEqual(template.description, self.finding.description) + self.assertEqual(template.mitigation, self.finding.mitigation) + self.assertEqual(template.impact, self.finding.impact) + self.assertEqual(template.references, self.finding.references) + + def test_mktemplate_fails_when_template_exists(self): + """Test that mktemplate fails when a template with the same title already exists""" + # Create a template with the same title first + existing_template = Finding_Template() + existing_template.title = self.finding.title + existing_template.cwe = 0 + existing_template.severity = "Low" + existing_template.save() + + # Try to create template from finding + result = self.make_request(self.user, self.finding.id) + + # Verify redirect (still redirects but with error message) + self.assertEqual(result.status_code, 302) + self.assertEqual(result.url, f"/finding/{self.finding.id}") + + # Verify only one template exists (the original one) + templates = Finding_Template.objects.filter(title=self.finding.title) + self.assertEqual(templates.count(), 1) + self.assertEqual(templates.first().id, existing_template.id) + + def test_mktemplate_requires_permission(self): + """Test that mktemplate requires Finding_Add permission""" + user = FindingTemplateTestUtil.create_user(is_staff=False) + user.is_superuser = False + user.save() + + # Should raise PermissionDenied + with self.assertRaises(PermissionDenied): + self.make_request(user, self.finding.id) + + +class TestAddFindingFromTemplate(DojoTestCase): + fixtures = ["dojo_testdata.json"] + + def setUp(self): + self.test = FindingMother.create().test + self.template = FindingTemplateMother.create() + self.user = FindingTemplateTestUtil.create_user(is_staff=True) + self.user.is_superuser = True + self.user.save() + # Add user as product member with Maintainer role (has Finding_Add permission) + maintainer_role = Role.objects.get(name="Maintainer") + Product_Member(user=self.user, product=self.test.engagement.product, role=maintainer_role).save() + + def make_get_request(self, user, test_id, template_id): + rf = RequestFactory() + request = rf.get(f"/test/{test_id}/add_findings/{template_id}") + request.user = user + request.session = {} + messages = FallbackStorage(request) + request._messages = messages + with impersonate(user): + return test_views.add_finding_from_template(request, tid=test_id, fid=template_id) + + def make_post_request(self, user, test_id, template_id, data=None): + rf = RequestFactory() + if data is None: + data = { + "title": self.template.title, + "date": timezone.now().date(), + "severity": self.template.severity, + "description": self.template.description, + "mitigation": self.template.mitigation or "", + "impact": self.template.impact or "", + "references": self.template.references or "", + "active": True, + "verified": True, + "false_p": False, + "duplicate": False, + "out_of_scope": False, + } + request = rf.post(f"/test/{test_id}/add_findings/{template_id}", data) + request.user = user + request.session = {} + messages = FallbackStorage(request) + request._messages = messages + with impersonate(user): + return test_views.add_finding_from_template(request, tid=test_id, fid=template_id) + + def test_add_finding_from_template_renders_form(self): + """Test that GET request renders the form with template data""" + result = self.make_get_request(self.user, self.test.id, self.template.id) + self.assertEqual(result.status_code, 200) + self.assertContains(result, self.template.title) + + def test_add_finding_from_template_creates_finding(self): + """Test that POST request creates a new finding from template""" + initial_count = Finding.objects.filter(test=self.test).count() + + result = self.make_post_request(self.user, self.test.id, self.template.id) + + # Should redirect to test view + self.assertEqual(result.status_code, 302) + self.assertEqual(result.url, f"/test/{self.test.id}") + + # Verify finding was created + final_count = Finding.objects.filter(test=self.test).count() + self.assertEqual(final_count, initial_count + 1) + + # Verify finding has template data + finding = Finding.objects.filter(test=self.test).order_by("-id").first() + # Note: title casing may vary, so just check it contains the template title + self.assertIn(self.template.title.lower(), finding.title.lower()) + self.assertEqual(finding.cwe, self.template.cwe) + self.assertEqual(finding.severity, self.template.severity) + self.assertEqual(finding.description, self.template.description) + self.assertEqual(finding.mitigation, self.template.mitigation or "") + self.assertEqual(finding.impact, self.template.impact or "") + self.assertEqual(finding.references, self.template.references or "") + + def test_add_finding_from_template_copies_all_fields(self): + """Test that all template fields are copied to the finding""" + # Update template with all new fields + self.template.cvssv3_score = 7.5 + self.template.cvssv4_score = 8.0 + self.template.fix_available = True + self.template.fix_version = "1.2.3" + self.template.planned_remediation_version = "1.3.0" + self.template.effort_for_fixing = "Low" + self.template.steps_to_reproduce = "Step 1: Do this\nStep 2: Do that" + self.template.severity_justification = "This is critical because..." + self.template.component_name = "test-component" + self.template.component_version = "1.0.0" + self.template.notes = "Template note content" + self.template.save() + + # Set vulnerability IDs + save_vulnerability_ids_template(self.template, ["CVE-2023-1234", "CVE-2023-5678"]) + + # Set endpoints + save_endpoints_template(self.template, ["https://example.com/api", "https://example.com/admin"]) + + result = self.make_post_request(self.user, self.test.id, self.template.id) + self.assertEqual(result.status_code, 302) + + finding = Finding.objects.filter(test=self.test).order_by("-id").first() + + # Verify all fields were copied + self.assertEqual(finding.cvssv3_score, 7.5) + self.assertEqual(finding.cvssv4_score, 8.0) + self.assertEqual(finding.fix_available, True) + self.assertEqual(finding.fix_version, "1.2.3") + self.assertEqual(finding.planned_remediation_version, "1.3.0") + self.assertEqual(finding.effort_for_fixing, "Low") + self.assertEqual(finding.steps_to_reproduce, "Step 1: Do this\nStep 2: Do that") + self.assertEqual(finding.severity_justification, "This is critical because...") + self.assertEqual(finding.component_name, "test-component") + self.assertEqual(finding.component_version, "1.0.0") + + # Verify vulnerability IDs were copied + vulnerability_ids = [vid.vulnerability_id for vid in Vulnerability_Id.objects.filter(finding=finding)] + self.assertIn("CVE-2023-1234", vulnerability_ids) + self.assertIn("CVE-2023-5678", vulnerability_ids) + + # Verify endpoints were copied + self.assertTrue(any("example.com/api" in str(ep) for ep in finding.endpoints.all())) + self.assertTrue(any("example.com/admin" in str(ep) for ep in finding.endpoints.all())) + + # Verify note was created + notes = Notes.objects.filter(finding=finding) + self.assertTrue(notes.exists()) + note = notes.first() + self.assertEqual(note.entry, "Template note content") + + def test_add_finding_from_template_requires_permission(self): + """Test that add_finding_from_template requires Finding_Add permission""" + unauthorized_user = FindingTemplateTestUtil.create_user(is_staff=False) + unauthorized_user.is_superuser = False + unauthorized_user.save() + + # Should raise PermissionDenied + with self.assertRaises(PermissionDenied): + self.make_get_request(unauthorized_user, self.test.id, self.template.id) + + def test_add_finding_from_template_updates_template_last_used(self): + """Test that template.last_used is updated when creating finding""" + original_last_used = self.template.last_used + + result = self.make_post_request(self.user, self.test.id, self.template.id) + self.assertEqual(result.status_code, 302) + + # Refresh template from database + self.template.refresh_from_db() + self.assertIsNotNone(self.template.last_used) + if original_last_used: + self.assertGreaterEqual(self.template.last_used, original_last_used) diff --git a/unittests/test_auditlog.py b/unittests/test_auditlog.py index c748758cc67..2af68cf76c6 100644 --- a/unittests/test_auditlog.py +++ b/unittests/test_auditlog.py @@ -1,23 +1,18 @@ """ Unit tests for audit configuration functionality. -Tests the dual-audit system where both django-auditlog and django-pghistory -can coexist, allowing users to see historical data from both systems. +Tests pghistory audit system configuration and event creation. """ -from unittest.mock import MagicMock, patch +import os +from unittest.mock import patch -from auditlog.models import LogEntry from django.apps import apps +from django.conf import settings from django.test import TestCase, override_settings from dojo.auditlog import ( configure_audit_system, configure_pghistory_triggers, - disable_django_auditlog, - disable_django_pghistory, - enable_django_auditlog, - enable_django_pghistory, - register_django_pghistory_models, ) from dojo.models import Product_Type @@ -26,103 +21,39 @@ class TestAuditConfig(TestCase): """Test audit configuration functionality.""" - @patch("auditlog.registry.auditlog") - def test_enable_django_auditlog(self, mock_auditlog): - """Test that enable_django_auditlog registers models.""" - # Mock the auditlog registry - mock_auditlog.register = MagicMock() - - enable_django_auditlog() - - # Verify that register was called multiple times (once for each model) - self.assertTrue(mock_auditlog.register.called) - self.assertGreater(mock_auditlog.register.call_count, 5) - - def test_disable_django_auditlog(self): - """Test that disable_django_auditlog runs without error.""" - # This should not raise an exception - disable_django_auditlog() - - @patch("dojo.auditlog.pghistory") - def test_register_django_pghistory_models(self, mock_pghistory): - """Test that register_django_pghistory_models registers all models.""" - # Mock pghistory.track - mock_pghistory.track = MagicMock() - mock_pghistory.InsertEvent = MagicMock() - mock_pghistory.UpdateEvent = MagicMock() - mock_pghistory.DeleteEvent = MagicMock() - mock_pghistory.ManualEvent = MagicMock() - - register_django_pghistory_models() - - # Verify that track was called multiple times (once for each model) - self.assertTrue(mock_pghistory.track.called) - self.assertGreater(mock_pghistory.track.call_count, 5) - @patch("dojo.auditlog.call_command") - def test_enable_django_pghistory(self, mock_call_command): - """Test that enable_django_pghistory enables triggers only.""" - enable_django_pghistory() + def test_configure_pghistory_triggers_enabled(self, mock_call_command): + """Test that configure_pghistory_triggers enables triggers when audit logging is enabled.""" + with override_settings(ENABLE_AUDITLOG=True): + configure_pghistory_triggers() # Verify that pgtrigger enable command was called mock_call_command.assert_called_with("pgtrigger", "enable") @patch("dojo.auditlog.call_command") - def test_disable_django_pghistory(self, mock_call_command): - """Test that disable_django_pghistory disables triggers.""" - disable_django_pghistory() + def test_configure_pghistory_triggers_disabled(self, mock_call_command): + """Test that configure_pghistory_triggers disables triggers when audit logging is disabled.""" + with override_settings(ENABLE_AUDITLOG=False): + configure_pghistory_triggers() # Verify that pgtrigger disable command was called - mock_call_command.assert_called_once_with("pgtrigger", "disable") - - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="invalid-type") - @patch("dojo.auditlog.disable_django_auditlog") - @patch("dojo.auditlog.call_command") - def test_invalid_audit_type_warning(self, mock_call_command, mock_disable_auditlog): - """Test that invalid audit types disable both audit systems.""" - # Call the main configuration function with invalid type - configure_audit_system() - configure_pghistory_triggers() - - # Verify that auditlog is disabled for invalid type - mock_disable_auditlog.assert_called_once() - # Verify that pghistory triggers are also disabled for invalid type mock_call_command.assert_called_with("pgtrigger", "disable") - # This test mainly ensures no exceptions are raised - - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-pghistory") - @patch("dojo.auditlog.enable_django_auditlog") - @patch("dojo.auditlog.disable_django_auditlog") - @patch("dojo.auditlog.call_command") - def test_dual_audit_system_coexistence(self, mock_call_command, mock_disable_auditlog, mock_enable_auditlog): - """Test that audit system configuration handles pghistory type correctly.""" - # Call the main configuration function + @override_settings(ENABLE_AUDITLOG=True) + def test_configure_audit_system_enabled(self): + """Test that configure_audit_system configures pghistory when audit logging is enabled.""" + # Should not raise an exception configure_audit_system() - configure_pghistory_triggers() - - # Verify auditlog is disabled when pghistory is the chosen type - mock_disable_auditlog.assert_called_once() - # Verify auditlog is not enabled when pghistory is chosen - mock_enable_auditlog.assert_not_called() - # Verify that pghistory triggers are enabled when pghistory is the chosen type - mock_call_command.assert_called_with("pgtrigger", "enable") - - # This demonstrates that the system correctly chooses the configured audit type - - def test_separate_history_lists_approach(self): - """Test that the dual-history approach creates separate lists correctly.""" - # This test verifies the new approach where we maintain separate history lists - # instead of mixing audit data from different systems - - # Import the view function to test the separation logic - # This is more of a structural test to ensure the approach is sound - # The actual view testing would require more complex setup + @override_settings(ENABLE_AUDITLOG=False) + def test_configure_audit_system_disabled(self): + """Test that configure_audit_system handles disabled audit logging.""" + # Should not raise an exception + configure_audit_system() - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-pghistory") + @override_settings(ENABLE_AUDITLOG=True) def test_pghistory_insert_event_creation(self): - """Test that pghistory creates insert events when a Product_Type is created and auditlog does not.""" + """Test that pghistory creates insert events when a Product_Type is created.""" # Configure audit system for pghistory configure_audit_system() configure_pghistory_triggers() @@ -133,9 +64,6 @@ def test_pghistory_insert_event_creation(self): # Count existing events before creating new Product_Type initial_event_count = ProductTypeEvent.objects.count() - # Clear any existing audit log entries for Product_Type - LogEntry.objects.filter(content_type__model="product_type").delete() - # Create a new Product_Type product_type = Product_Type.objects.create( name="Test Product Type for pghistory", @@ -159,139 +87,41 @@ def test_pghistory_insert_event_creation(self): "Event should contain the Product_Type description") # Verify it's an insert event (check if pgh_label indicates creation) - # The label could be 'insert' or contain insert-related information self.assertIsNotNone(latest_event.pgh_created_at, "Event should have a creation timestamp") - # Verify that NO auditlog entries were created (mutual exclusivity) - audit_entries = LogEntry.objects.filter( - content_type__model="product_type", - object_id=product_type.id, - ) - self.assertEqual(audit_entries.count(), 0, - "Expected NO auditlog entries when pghistory is enabled") - # Clean up product_type.delete() - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-auditlog") - @patch("dojo.auditlog.enable_django_auditlog") - @patch("dojo.auditlog.call_command") - def test_configure_audit_system_auditlog_enabled(self, mock_call_command, mock_enable_auditlog): - """Test that configure_audit_system enables auditlog and configures pghistory triggers correctly.""" - configure_audit_system() - configure_pghistory_triggers() - - # Verify that auditlog is enabled - mock_enable_auditlog.assert_called_once() - # Verify that pghistory triggers are disabled when auditlog is the chosen type - mock_call_command.assert_called_with("pgtrigger", "disable") - - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-pghistory") - @patch("dojo.auditlog.disable_django_auditlog") - @patch("dojo.auditlog.call_command") - def test_configure_audit_system_pghistory_enabled(self, mock_call_command, mock_disable_auditlog): - """Test that configure_audit_system disables auditlog and enables pghistory triggers correctly.""" - configure_audit_system() - configure_pghistory_triggers() - - # Verify that auditlog is disabled when pghistory is the chosen type - mock_disable_auditlog.assert_called_once() - # Verify that pghistory triggers are enabled when pghistory is the chosen type - mock_call_command.assert_called_with("pgtrigger", "enable") - - @override_settings(ENABLE_AUDITLOG=False) - @patch("dojo.auditlog.disable_django_auditlog") - @patch("dojo.auditlog.call_command") - def test_configure_audit_system_all_disabled(self, mock_call_command, mock_disable_auditlog): - """Test that configure_audit_system disables both auditlog and pghistory when audit is disabled.""" - configure_audit_system() - configure_pghistory_triggers() - - # Verify that auditlog is disabled when audit logging is disabled - mock_disable_auditlog.assert_called_once() - # Verify that pghistory triggers are also disabled when audit logging is disabled - mock_call_command.assert_called_with("pgtrigger", "disable") - - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="unknown-type") - @patch("dojo.auditlog.disable_django_auditlog") - @patch("dojo.auditlog.call_command") - def test_configure_audit_system_unknown_type(self, mock_call_command, mock_disable_auditlog): - """Test that configure_audit_system disables both systems for unknown audit types.""" - configure_audit_system() - configure_pghistory_triggers() - - # Verify that auditlog is disabled for unknown types - mock_disable_auditlog.assert_called_once() - # Verify that pghistory triggers are also disabled for unknown types - mock_call_command.assert_called_with("pgtrigger", "disable") - - @patch("dojo.auditlog.call_command") - def test_disable_pghistory_command_failure(self, mock_call_command): - """Test that disable_django_pghistory handles command failures gracefully.""" - # Simulate command failure - mock_call_command.side_effect = Exception("Command failed") - - # This should not raise an exception - disable_django_pghistory() - - # Verify that call_command was attempted - mock_call_command.assert_called_once_with("pgtrigger", "disable") - - @patch("dojo.auditlog.call_command") - def test_enable_pghistory_command_failure(self, mock_call_command): - """Test that enable_django_pghistory handles command failures gracefully.""" - # Simulate command failure for trigger enable - mock_call_command.side_effect = Exception("Command failed") - - # This should not raise an exception - enable_django_pghistory() - - # Verify that call_command was attempted - mock_call_command.assert_called_with("pgtrigger", "enable") - - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-auditlog") - def test_auditlog_insert_event_creation(self): - """Test that django-auditlog creates audit log entries when a Product_Type is created and pghistory does not.""" - # Configure audit system for auditlog - configure_audit_system() - configure_pghistory_triggers() - - # Get the Product_Type event model for pghistory check - ProductTypeEvent = apps.get_model("dojo", "Product_TypeEvent") - - # Clear any existing audit log entries for Product_Type - LogEntry.objects.filter(content_type__model="product_type").delete() - - # Count existing pghistory events - initial_pghistory_count = ProductTypeEvent.objects.count() - - # Create a new Product_Type - product_type = Product_Type.objects.create( - name="Test Product Type for Auditlog", - description="Test description for auditlog verification", - ) - - # Verify that an audit log entry was created - audit_entries = LogEntry.objects.filter( - content_type__model="product_type", - object_id=product_type.id, - action=LogEntry.Action.CREATE, - ) - - self.assertEqual(audit_entries.count(), 1, - "Expected exactly one audit log entry for Product_Type creation") - - audit_entry = audit_entries.first() - self.assertEqual(audit_entry.object_repr, str(product_type), - "Audit entry should represent the created object") - self.assertIsNotNone(audit_entry.timestamp, - "Audit entry should have a timestamp") - - # Verify that NO pghistory events were created (mutual exclusivity) - final_pghistory_count = ProductTypeEvent.objects.count() - self.assertEqual(final_pghistory_count, initial_pghistory_count, - "Expected NO new pghistory events when auditlog is enabled") - - # Clean up - product_type.delete() + def test_configure_audit_system_fails_with_dd_auditlog_type_env(self): + """Test that configure_audit_system fails if DD_AUDITLOG_TYPE environment variable is set.""" + # Temporarily set the environment variable + original_value = os.environ.get("DD_AUDITLOG_TYPE") + try: + os.environ["DD_AUDITLOG_TYPE"] = "django-pghistory" + with self.assertRaises(ValueError) as context: + configure_audit_system() + self.assertIn("DD_AUDITLOG_TYPE", str(context.exception)) + finally: + # Restore original value + if original_value is None: + os.environ.pop("DD_AUDITLOG_TYPE", None) + else: + os.environ["DD_AUDITLOG_TYPE"] = original_value + + def test_configure_audit_system_fails_with_auditlog_type_setting(self): + """Test that configure_audit_system fails if AUDITLOG_TYPE setting is manually set.""" + # Temporarily add the setting + original_value = getattr(settings, "AUDITLOG_TYPE", None) + try: + settings.AUDITLOG_TYPE = "django-pghistory" + with self.assertRaises(ValueError) as context: + configure_audit_system() + self.assertIn("AUDITLOG_TYPE", str(context.exception)) + finally: + # Restore original value + if original_value is None: + if hasattr(settings, "AUDITLOG_TYPE"): + delattr(settings, "AUDITLOG_TYPE") + else: + settings.AUDITLOG_TYPE = original_value diff --git a/unittests/test_bulk_edit_validation.py b/unittests/test_bulk_edit_validation.py new file mode 100644 index 00000000000..84ce98e396f --- /dev/null +++ b/unittests/test_bulk_edit_validation.py @@ -0,0 +1,746 @@ +import datetime + +from django.contrib.auth.models import User +from django.contrib.messages import get_messages +from django.urls import reverse +from django.utils import timezone + +from dojo.authorization.roles_permissions import Roles +from dojo.models import ( + Engagement, + Finding, + Product, + Product_Type, + Product_Type_Member, + Role, + Test, + Test_Type, +) + +from .dojo_test_case import DojoTestCase + + +class TestBulkEditValidation(DojoTestCase): + + """Test bulk edit validation rules for issue #11336""" + + @classmethod + def setUpTestData(cls): + # Create user with permissions + cls.user = User(username="testuser", is_staff=True) + cls.user.set_password("testpass") + cls.user.save() + + # Create product type and product + cls.product_type = Product_Type.objects.create(name="Web App") + cls.product = Product.objects.create( + prod_type=cls.product_type, + name="Test Product", + description="Test product for bulk edit validation", + ) + # Give user owner role + Product_Type_Member.objects.create( + product_type=cls.product_type, + user=cls.user, + role=Role.objects.get(id=Roles.Owner), + ) + + # Create engagement and test + cls.engagement = Engagement.objects.create( + product=cls.product, + target_start=timezone.now(), + target_end=timezone.now() + datetime.timedelta(days=30), + ) + cls.test_type = Test_Type.objects.create( + name="Manual Test", + static_tool=True, + ) + cls.test = Test.objects.create( + engagement=cls.engagement, + test_type=cls.test_type, + target_start=timezone.now(), + target_end=timezone.now() + datetime.timedelta(days=1), + ) + + # Create findings in various states + # Normal finding (not duplicate, not active) + cls.normal_finding = Finding.objects.create( + test=cls.test, + title="Normal Finding", + severity="High", + active=False, + verified=False, + duplicate=False, + reporter=cls.user, + numerical_severity="S1", + ) + + # Duplicate finding (duplicate=True, active=False) + cls.original_finding = Finding.objects.create( + test=cls.test, + title="Original Finding", + severity="High", + active=True, + verified=True, + duplicate=False, + reporter=cls.user, + numerical_severity="S1", + ) + cls.duplicate_finding = Finding.objects.create( + test=cls.test, + title="Duplicate Finding", + severity="High", + active=False, + verified=False, + duplicate=True, + duplicate_finding=cls.original_finding, + reporter=cls.user, + numerical_severity="S1", + ) + + # Active finding (active=True, not duplicate) + cls.active_finding = Finding.objects.create( + test=cls.test, + title="Active Finding", + severity="High", + active=True, + verified=True, + duplicate=False, + reporter=cls.user, + numerical_severity="S1", + ) + + # Inactive finding (active=False, not duplicate) + cls.inactive_finding = Finding.objects.create( + test=cls.test, + title="Inactive Finding", + severity="High", + active=False, + verified=False, + duplicate=False, + reporter=cls.user, + numerical_severity="S1", + ) + + def setUp(self): + self.client.force_login(self.user) + + def _bulk_edit_post_data(self, finding_ids, **kwargs): + """Helper to build POST data for bulk edit""" + data = { + "finding_to_update": [str(fid) for fid in finding_ids], + "return_url": reverse("view_test", args=(self.test.id,)), + } + # Add status checkbox if any status fields are being set + if any( + key in kwargs + for key in [ + "active", + "verified", + "false_p", + "out_of_scope", + "is_mitigated", + "under_review", + ] + ): + data["status"] = "on" + + # Add status fields + if kwargs.get("active"): + data["active"] = "on" + if kwargs.get("verified"): + data["verified"] = "on" + if kwargs.get("false_p"): + data["false_p"] = "on" + if kwargs.get("out_of_scope"): + data["out_of_scope"] = "on" + if kwargs.get("is_mitigated"): + data["is_mitigated"] = "on" + if kwargs.get("under_review"): + data["under_review"] = "on" + if kwargs.get("duplicate"): + data["duplicate"] = "on" + + # Add risk acceptance fields + if kwargs.get("risk_acceptance"): + data["risk_acceptance"] = "on" + if kwargs.get("risk_accept"): + data["risk_accept"] = "on" + if kwargs.get("risk_unaccept"): + data["risk_unaccept"] = "on" + + # Add other fields + if "severity" in kwargs: + data["severity"] = kwargs["severity"] + if "date" in kwargs: + data["date"] = kwargs["date"] + + return data + + def _assert_finding_status(self, finding, **expected_fields): + """Helper to verify finding state""" + finding.refresh_from_db() + for field, expected_value in expected_fields.items(): + actual_value = getattr(finding, field) + self.assertEqual( + actual_value, + expected_value, + f"Finding {finding.id} field {field}: expected {expected_value}, got {actual_value}", + ) + + def _get_messages_text(self, response): + """Helper to get all message texts from response""" + # Django test client stores messages in the session + # Try multiple methods to access them + messages_list = [] + try: + # Method 1: Try via wsgi_request if available + if hasattr(response, "wsgi_request") and response.wsgi_request: + messages_list = [str(m) for m in get_messages(response.wsgi_request)] + # Method 2: Try via response.request._messages + elif hasattr(response, "request") and hasattr(response.request, "_messages"): + storage = response.request._messages + messages_list = [str(m) for m in storage] + # Method 3: Try via client session (Django test client stores messages here) + elif hasattr(response, "client") and hasattr(response.client, "session"): + # Messages are stored in the session + # Create a mock request to access messages from session + from django.test import RequestFactory # noqa: PLC0415 + + factory = RequestFactory() + request = factory.get("/") + request.session = response.client.session + messages_list = [str(m) for m in get_messages(request)] + except (AttributeError, TypeError, ImportError): + # If messages aren't accessible, that's okay + # Tests can still verify behavior by checking finding state + pass + return messages_list + + # Form Validation Tests + + def test_form_rejects_active_and_risk_accept_together(self): + """Test that form validation rejects active + risk_accept""" + from dojo.forms import FindingBulkUpdateForm # noqa: PLC0415 + form_data = { + "active": True, + "risk_acceptance": True, + "risk_accept": True, + "severity": "", + "verified": False, + "false_p": False, + "duplicate": False, + "out_of_scope": False, + "under_review": False, + "is_mitigated": False, + } + form = FindingBulkUpdateForm(data=form_data) + self.assertFalse(form.is_valid()) + self.assertIn("Active findings cannot be risk accepted", str(form.errors)) + + def test_form_rejects_duplicate_and_active_together(self): + """Test that form validation rejects duplicate + active""" + from dojo.forms import FindingBulkUpdateForm # noqa: PLC0415 + form_data = { + "duplicate": True, + "active": True, + "severity": "", + "verified": False, + "false_p": False, + "out_of_scope": False, + "under_review": False, + "is_mitigated": False, + } + form = FindingBulkUpdateForm(data=form_data) + self.assertFalse(form.is_valid()) + self.assertIn("Duplicate findings cannot be verified or active", str(form.errors)) + + def test_form_rejects_duplicate_and_verified_together(self): + """Test that form validation rejects duplicate + verified""" + from dojo.forms import FindingBulkUpdateForm # noqa: PLC0415 + form_data = { + "duplicate": True, + "verified": True, + "severity": "", + "active": False, + "false_p": False, + "out_of_scope": False, + "under_review": False, + "is_mitigated": False, + } + form = FindingBulkUpdateForm(data=form_data) + self.assertFalse(form.is_valid()) + self.assertIn("Duplicate findings cannot be verified or active", str(form.errors)) + + def test_form_allows_duplicate_with_other_fields(self): + """Test that form allows duplicate with other non-conflicting fields""" + from dojo.forms import FindingBulkUpdateForm # noqa: PLC0415 + form_data = { + "duplicate": True, + "false_p": True, + "out_of_scope": True, + "severity": "", + "active": False, + "verified": False, + "under_review": False, + "is_mitigated": False, + } + form = FindingBulkUpdateForm(data=form_data) + # Form should be valid (though it may have other validation issues) + # The key is that duplicate + false_p + out_of_scope doesn't raise our specific error + form_errors = str(form.errors) if not form.is_valid() else "" + self.assertNotIn("Duplicate findings cannot be verified or active", form_errors) + + # View-Level Validation Tests (Duplicate Findings) + + def test_bulk_edit_duplicate_finding_cannot_set_active(self): + """Test that duplicate findings cannot be set as active via bulk edit""" + post_data = self._bulk_edit_post_data( + [self.duplicate_finding.id], + active=True, + false_p=False, + out_of_scope=False, + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify finding remains inactive + self._assert_finding_status( + self.duplicate_finding, + active=False, + duplicate=True, + ) + # Verify other fields can still be updated + # (We set false_p=False, so it should remain False) + + # Verify warning message + messages = self._get_messages_text(response) + warning_messages = [m for m in messages if "duplicate findings" in m.lower()] + self.assertGreater( + len(warning_messages), + 0, + f"Expected warning about duplicate findings, got messages: {messages}", + ) + self.assertIn("Skipped status update", warning_messages[0]) + + def test_bulk_edit_duplicate_finding_cannot_set_verified(self): + """Test that duplicate findings cannot be set as verified via bulk edit""" + post_data = self._bulk_edit_post_data( + [self.duplicate_finding.id], + verified=True, + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify finding remains unverified + self._assert_finding_status( + self.duplicate_finding, + verified=False, + duplicate=True, + ) + + # Verify warning message + messages = self._get_messages_text(response) + warning_messages = [m for m in messages if "duplicate findings" in m.lower()] + self.assertGreater(len(warning_messages), 0) + + def test_bulk_edit_duplicate_finding_can_update_other_fields(self): + """Test that duplicate findings can update other status fields""" + post_data = self._bulk_edit_post_data( + [self.duplicate_finding.id], + false_p=True, + out_of_scope=True, + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify other fields are updated + self._assert_finding_status( + self.duplicate_finding, + false_p=True, + out_of_scope=True, + duplicate=True, + active=False, # Should remain False + ) + + # Verify no warning about duplicates (no conflict) + messages = self._get_messages_text(response) + warning_messages = [m for m in messages if "duplicate findings" in m.lower()] + self.assertEqual( + len(warning_messages), + 0, + f"Unexpected duplicate warning: {warning_messages}", + ) + + def test_bulk_edit_duplicate_finding_severity_update_works(self): + """Test that severity can be updated on duplicate findings""" + post_data = self._bulk_edit_post_data( + [self.duplicate_finding.id], + severity="Critical", + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify severity is updated + self._assert_finding_status( + self.duplicate_finding, + severity="Critical", + duplicate=True, + ) + + # Verify no validation errors + self.assertNotEqual(response.status_code, 500) + + # View-Level Validation Tests (Active + Risk Acceptance) + + def test_bulk_edit_active_finding_cannot_accept_risk(self): + """Test that active findings cannot accept risk via bulk edit""" + # Enable simple risk acceptance on product + self.product.enable_simple_risk_acceptance = True + self.product.save() + + post_data = self._bulk_edit_post_data( + [self.active_finding.id], + risk_acceptance=True, + risk_accept=True, + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify finding is NOT risk accepted + self.active_finding.refresh_from_db() + self.assertFalse( + self.active_finding.risk_accepted, + "Active finding should not be risk accepted", + ) + + # Verify warning message + messages = self._get_messages_text(response) + warning_messages = [ + m for m in messages if "active findings" in m.lower() and "risk" in m.lower() + ] + self.assertGreater( + len(warning_messages), + 0, + f"Expected warning about active findings and risk acceptance, got: {messages}", + ) + + def test_bulk_edit_inactive_finding_can_accept_risk(self): + """Test that inactive findings can accept risk""" + # Enable simple risk acceptance on product + self.product.enable_simple_risk_acceptance = True + self.product.save() + + post_data = self._bulk_edit_post_data( + [self.inactive_finding.id], + risk_acceptance=True, + risk_accept=True, + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify finding IS risk accepted + self.inactive_finding.refresh_from_db() + self.assertTrue( + self.inactive_finding.risk_accepted, + "Inactive finding should be risk accepted", + ) + + # Verify no warning about active findings + messages = self._get_messages_text(response) + warning_messages = [ + m for m in messages if "active findings" in m.lower() and "risk" in m.lower() + ] + self.assertEqual( + len(warning_messages), + 0, + f"Unexpected active findings warning: {warning_messages}", + ) + + def test_bulk_edit_duplicate_finding_can_accept_risk_if_not_active(self): + """Test that duplicate but inactive findings can accept risk""" + # Enable simple risk acceptance on product + self.product.enable_simple_risk_acceptance = True + self.product.save() + + post_data = self._bulk_edit_post_data( + [self.duplicate_finding.id], + risk_acceptance=True, + risk_accept=True, + ) + + self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify finding IS risk accepted (duplicate check happens first, then active check) + self.duplicate_finding.refresh_from_db() + self.assertTrue( + self.duplicate_finding.risk_accepted, + "Duplicate but inactive finding should be risk accepted", + ) + + # User Feedback Tests + + def test_bulk_edit_shows_success_message_with_actual_count(self): + """Test that success message shows actually_updated_count""" + # Create mix: 1 duplicate, 2 normal findings + normal1 = Finding.objects.create( + test=self.test, + title="Normal 1", + severity="High", + active=False, + reporter=self.user, + numerical_severity="S1", + ) + normal2 = Finding.objects.create( + test=self.test, + title="Normal 2", + severity="High", + active=False, + reporter=self.user, + numerical_severity="S1", + ) + + post_data = self._bulk_edit_post_data( + [self.duplicate_finding.id, normal1.id, normal2.id], + active=True, + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify success message + messages = self._get_messages_text(response) + success_messages = [m for m in messages if "successful" in m.lower()] + self.assertGreater(len(success_messages), 0) + + # Verify warning about skipped duplicate + warning_messages = [m for m in messages if "duplicate findings" in m.lower()] + self.assertGreater(len(warning_messages), 0) + + # Verify normal findings were updated + self._assert_finding_status(normal1, active=True) + self._assert_finding_status(normal2, active=True) + + def test_bulk_edit_shows_multiple_warning_messages(self): + """Test that multiple warning messages appear for different conflicts""" + # Enable simple risk acceptance + self.product.enable_simple_risk_acceptance = True + self.product.save() + + # First, try to set duplicate finding as active (will be skipped) + post_data1 = self._bulk_edit_post_data( + [self.duplicate_finding.id], + active=True, # Will conflict with duplicate + ) + response1 = self.client.post( + reverse("finding_bulk_update_all"), + post_data1, + follow=True, + ) + + # Then, try to risk accept active finding (will be skipped) + post_data2 = self._bulk_edit_post_data( + [self.active_finding.id], + risk_acceptance=True, + risk_accept=True, # Will conflict with active + ) + response2 = self.client.post( + reverse("finding_bulk_update_all"), + post_data2, + follow=True, + ) + + # Combine messages from both requests + messages1 = self._get_messages_text(response1) + messages2 = self._get_messages_text(response2) + all_messages = messages1 + messages2 + + duplicate_warnings = [ + m for m in all_messages if "duplicate findings" in m.lower() + ] + active_warnings = [ + m + for m + in all_messages + if "active findings" in m.lower() and "risk" in m.lower() + ] + + self.assertGreater( + len(duplicate_warnings), + 0, + f"Expected duplicate warning, got: {all_messages}", + ) + self.assertGreater( + len(active_warnings), + 0, + f"Expected active risk acceptance warning, got: {all_messages}", + ) + + def test_bulk_edit_no_warning_when_no_conflicts(self): + """Test that no warnings appear when there are no conflicts""" + post_data = self._bulk_edit_post_data( + [self.normal_finding.id], + active=True, + verified=True, + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + messages = self._get_messages_text(response) + warning_messages = [ + m + for m + in messages + if "duplicate" in m.lower() or ("active" in m.lower() and "risk" in m.lower()) + ] + + self.assertEqual( + len(warning_messages), + 0, + f"Unexpected warnings: {warning_messages}", + ) + + # Verify success message + success_messages = [m for m in messages if "successful" in m.lower()] + self.assertGreater(len(success_messages), 0) + + # Edge Cases + + def test_bulk_edit_mixed_findings_partial_success(self): + """Test bulk edit with mix of duplicate and normal findings""" + # Create 2 more normal findings + normal1 = Finding.objects.create( + test=self.test, + title="Normal 1", + severity="High", + active=False, + reporter=self.user, + numerical_severity="S1", + ) + normal2 = Finding.objects.create( + test=self.test, + title="Normal 2", + severity="High", + active=False, + reporter=self.user, + numerical_severity="S1", + ) + normal3 = Finding.objects.create( + test=self.test, + title="Normal 3", + severity="High", + active=False, + reporter=self.user, + numerical_severity="S1", + ) + + post_data = self._bulk_edit_post_data( + [ + self.duplicate_finding.id, + self.duplicate_finding.id, # Add same duplicate twice to test + normal1.id, + normal2.id, + normal3.id, + ], + active=True, + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify normal findings were updated + self._assert_finding_status(normal1, active=True) + self._assert_finding_status(normal2, active=True) + self._assert_finding_status(normal3, active=True) + + # Verify duplicate findings remain inactive + self._assert_finding_status(self.duplicate_finding, active=False, duplicate=True) + + # Verify warning message shows skipped count + messages = self._get_messages_text(response) + warning_messages = [m for m in messages if "duplicate findings" in m.lower()] + self.assertGreater(len(warning_messages), 0) + + def test_bulk_edit_severity_only_no_status_conflicts(self): + """Test that severity-only updates work regardless of duplicate status""" + post_data = self._bulk_edit_post_data( + [self.duplicate_finding.id], + severity="Critical", + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify severity is updated + self._assert_finding_status( + self.duplicate_finding, + severity="Critical", + duplicate=True, + ) + + # Verify no validation errors or warnings + messages = self._get_messages_text(response) + warning_messages = [m for m in messages if "duplicate" in m.lower()] + self.assertEqual(len(warning_messages), 0) + + def test_bulk_edit_date_update_works_regardless_of_duplicate_status(self): + """Test that date updates work regardless of duplicate status""" + new_date = timezone.now().date() + post_data = self._bulk_edit_post_data( + [self.duplicate_finding.id], + date=new_date, + ) + + response = self.client.post( + reverse("finding_bulk_update_all"), + post_data, + follow=True, + ) + + # Verify date is updated + self.duplicate_finding.refresh_from_db() + self.assertEqual(self.duplicate_finding.date, new_date) + + # Verify no validation errors + self.assertNotEqual(response.status_code, 500) diff --git a/unittests/test_deduplication_logic.py b/unittests/test_deduplication_logic.py index f5390eaef60..118ca267f91 100644 --- a/unittests/test_deduplication_logic.py +++ b/unittests/test_deduplication_logic.py @@ -9,6 +9,7 @@ from django.core import serializers from django.utils import timezone +from dojo.finding.deduplication import set_duplicate from dojo.importers.default_importer import DefaultImporter from dojo.models import ( Development_Environment, @@ -879,6 +880,136 @@ def test_extra_endpoints_unique_id(self): # expect duplicate: unique_id match regardless of extra endpoints self.assert_finding(finding_new, not_pk=124, duplicate=True, duplicate_finding_id=124, hash_code=finding_124.hash_code) + def test_regression_duplicate_reopen_unique_id(self): + """ + Test the is_duplicate_reopen exception in set_duplicate. + When trying to attach a new active finding to a mitigated finding, + an exception should be raised. + + Related to: https://github.com/DefectDojo/django-DefectDojo/issues/14010 + """ + # Get an existing finding with unique_id (finding 124) + finding_124 = Finding.objects.get(id=124) + original_unique_id = finding_124.unique_id_from_tool + + # Mitigate the existing finding + finding_124.active = False + finding_124.is_mitigated = True + finding_124.mitigated = timezone.now() + finding_124.out_of_scope = False + finding_124.false_p = False + finding_124.save(dedupe_option=False) + + # Create a new finding with the same unique_id that is active (not mitigated) + finding_new, _ = self.copy_and_reset_finding(find_id=124) + finding_new.active = True + finding_new.is_mitigated = False + finding_new.mitigated = None + finding_new.unique_id_from_tool = original_unique_id + finding_new.save(dedupe_option=False) + + # Try to deduplicate - this should raise an exception + with self.assertRaises(Exception) as context: + set_duplicate(finding_new, finding_124) + + self.assertIn("Found a regression", str(context.exception)) + + def test_duplicate_attached_to_mitigated_finding_unique_id(self): + """ + Test the exception when attaching a finding that is already marked as duplicate + to a mitigated finding. + + Related to: https://github.com/DefectDojo/django-DefectDojo/issues/14010 + """ + # Get an existing finding with unique_id (finding 124) + finding_124 = Finding.objects.get(id=124) + original_unique_id = finding_124.unique_id_from_tool + + # Mitigate the existing finding + finding_124.active = False + finding_124.is_mitigated = True + finding_124.mitigated = timezone.now() + finding_124.save(dedupe_option=False) + + # Create a new finding that is already marked as duplicate AND also mitigated + # This ensures that is_duplicate_reopen doesn't trigger first + finding_new, _ = self.copy_and_reset_finding(find_id=124) + finding_new.duplicate = True + finding_new.active = False + finding_new.is_mitigated = True + finding_new.mitigated = timezone.now() + finding_new.unique_id_from_tool = original_unique_id + finding_new.save(dedupe_option=False) + + # Try to deduplicate - this should raise an exception + with self.assertRaises(Exception) as context: + set_duplicate(finding_new, finding_124) + + self.assertIn("Skip this finding as we do not want to attach a new duplicate to a mitigated finding", str(context.exception)) + + def test_multiple_findings_same_unique_id_mixed_states_unique_id(self): + """ + Test deduplication with multiple findings having the same unique_id, + where some are mitigated and some are active. The deduplication should + skip mitigated ones and use the first active one. + + Related to: https://github.com/DefectDojo/django-DefectDojo/issues/14010 + """ + # Get an existing finding with unique_id (finding 124) + finding_124 = Finding.objects.get(id=124) + original_unique_id = finding_124.unique_id_from_tool + + # Mitigate the original finding so it's not a candidate + finding_124.active = False + finding_124.is_mitigated = True + finding_124.mitigated = timezone.now() + finding_124.save() + + # Create multiple findings with the same unique_id + # First: active finding (this should become the original) + finding_active, _ = self.copy_and_reset_finding(find_id=124) + finding_active.active = True + finding_active.is_mitigated = False + finding_active.mitigated = None + finding_active.unique_id_from_tool = original_unique_id + finding_active.save(dedupe_option=False) # Don't deduplicate so it remains active + + # Second: mitigated finding + finding_mitigated, _ = self.copy_and_reset_finding(find_id=124) + finding_mitigated.active = False + finding_mitigated.is_mitigated = True + finding_mitigated.mitigated = timezone.now() + finding_mitigated.out_of_scope = False + finding_mitigated.false_p = False + finding_mitigated.unique_id_from_tool = original_unique_id + finding_mitigated.save(dedupe_option=False) + + # Ensure mitigated finding has lower ID (will be checked first) + if finding_mitigated.id > finding_active.id: + # Swap by deleting and recreating in correct order + finding_mitigated.delete() + finding_mitigated, _ = self.copy_and_reset_finding(find_id=124) + finding_mitigated.active = False + finding_mitigated.is_mitigated = True + finding_mitigated.mitigated = timezone.now() + finding_mitigated.out_of_scope = False + finding_mitigated.false_p = False + finding_mitigated.unique_id_from_tool = original_unique_id + finding_mitigated.save(dedupe_option=False) + + # Create a new finding with the same unique_id + # It should skip the mitigated finding and deduplicate against the active one + finding_new, _ = self.copy_and_reset_finding(find_id=124) + finding_new.active = True + finding_new.is_mitigated = False + finding_new.unique_id_from_tool = original_unique_id + finding_new.save() + + # The new finding should be marked as duplicate of the active finding, + # not the mitigated one (even though mitigated has lower ID) + self.assert_finding(finding_new, duplicate=True, duplicate_finding_id=finding_active.id) + self.assertNotEqual(finding_new.duplicate_finding.id, finding_mitigated.id) + # algo unique_id_or_hash_code Veracode scan def test_identical_unique_id_or_hash_code(self): diff --git a/unittests/test_finding_helper.py b/unittests/test_finding_helper.py index c6d1d65008d..7d92ab2992c 100644 --- a/unittests/test_finding_helper.py +++ b/unittests/test_finding_helper.py @@ -10,7 +10,7 @@ from rest_framework.test import APIClient from dojo.finding.helper import save_vulnerability_ids, save_vulnerability_ids_template -from dojo.models import Finding, Finding_Template, Test, Vulnerability_Id, Vulnerability_Id_Template +from dojo.models import Finding, Finding_Template, Test, Vulnerability_Id from .dojo_test_case import DojoAPITestCase, DojoTestCase @@ -233,19 +233,15 @@ def test_save_vulnerability_ids(self, save_mock, delete_mock, filter_mock): self.assertEqual(save_mock.call_count, 2) self.assertEqual("REF-1", finding.cve) - @patch("dojo.finding.helper.Vulnerability_Id_Template.objects.filter") - @patch("django.db.models.query.QuerySet.delete") - @patch("dojo.finding.helper.Vulnerability_Id_Template.save") - def test_save_vulnerability_id_templates(self, save_mock, delete_mock, filter_mock): + @patch("dojo.models.Finding_Template.save") + def test_save_vulnerability_id_templates(self, save_mock): finding_template = Finding_Template() new_vulnerability_ids = ["REF-1", "REF-2", "REF-2"] - filter_mock.return_value = Vulnerability_Id_Template.objects.none() save_vulnerability_ids_template(finding_template, new_vulnerability_ids) - filter_mock.assert_called_with(finding_template=finding_template) - delete_mock.assert_called_once() - self.assertEqual(save_mock.call_count, 2) + save_mock.assert_called_once() + self.assertEqual("REF-1\nREF-2", finding_template.vulnerability_ids_text) self.assertEqual("REF-1", finding_template.cve) diff --git a/unittests/test_flush_auditlog.py b/unittests/test_flush_auditlog.py index 1c7f5ef08df..da347d952d5 100644 --- a/unittests/test_flush_auditlog.py +++ b/unittests/test_flush_auditlog.py @@ -1,11 +1,16 @@ +""" +Unit tests for flush_auditlog functionality. + +Tests the flush_auditlog management command and task that removes old audit log entries. +""" import logging -from datetime import UTC, date, datetime +from datetime import UTC, datetime -from auditlog.models import LogEntry from dateutil.relativedelta import relativedelta +from django.apps import apps from django.test import override_settings -from dojo.models import Finding +from dojo.models import Product_Type from dojo.tasks import flush_auditlog from .dojo_test_case import DojoTestCase @@ -18,32 +23,71 @@ class TestFlushAuditlog(DojoTestCase): @override_settings(AUDITLOG_FLUSH_RETENTION_PERIOD=-1) def test_flush_auditlog_disabled(self): - entries_before = LogEntry.objects.all().count() + """Test that flush_auditlog does nothing when retention period is -1 (disabled).""" + # Get pghistory event model + ProductTypeEvent = apps.get_model("dojo", "Product_TypeEvent") + entries_before = ProductTypeEvent.objects.count() + flush_auditlog() - entries_after = LogEntry.objects.all().count() + + entries_after = ProductTypeEvent.objects.count() self.assertEqual(entries_before, entries_after) @override_settings(AUDITLOG_FLUSH_RETENTION_PERIOD=0) def test_delete_all_entries(self): - entries_before = LogEntry.objects.filter(timestamp__date__lt=date.today()).count() + """Test that flush_auditlog deletes all entries when retention period is 0.""" + # Get pghistory event model + ProductTypeEvent = apps.get_model("dojo", "Product_TypeEvent") + + # Create a test product type to generate events + product_type = Product_Type.objects.create( + name="Test Product Type for Flush", + description="Test description", + ) + + # Flush with retention period 0 (delete all) flush_auditlog() - entries_after = LogEntry.objects.filter(timestamp__date__lt=date.today()).count() - # we have three old log entries in our testdata - self.assertEqual(entries_before - 3, entries_after) + + # All entries should be deleted + entries_after = ProductTypeEvent.objects.count() + self.assertEqual(entries_after, 0, "All entries should be deleted when retention period is 0") + + # Clean up + product_type.delete() @override_settings(AUDITLOG_FLUSH_RETENTION_PERIOD=1) def test_delete_entries_with_retention_period(self): - entries_before = LogEntry.objects.filter(timestamp__date__lt=datetime.now(UTC)).count() - two_weeks_ago = datetime.now(UTC) - relativedelta(weeks=2) - log_entry = LogEntry.objects.log_create( - instance=Finding.objects.all()[0], - timestamp=two_weeks_ago, - changes="foo", - action=LogEntry.Action.UPDATE, + """Test that flush_auditlog deletes entries older than retention period.""" + # Get pghistory event model + ProductTypeEvent = apps.get_model("dojo", "Product_TypeEvent") + + # Create a test product type + product_type = Product_Type.objects.create( + name="Test Product Type for Retention", + description="Test description", ) - log_entry.timestamp = two_weeks_ago - log_entry.save() + + # Get the event created by the creation + recent_event = ProductTypeEvent.objects.filter(pgh_obj_id=product_type.id).first() + + # Manually create an old event by updating the timestamp + # Set it to 2 months ago so it will be deleted with retention period of 1 month + if recent_event: + two_months_ago = datetime.now(UTC) - relativedelta(months=2) + # Update the created_at timestamp to make it old + ProductTypeEvent.objects.filter(pk=recent_event.pk).update(pgh_created_at=two_months_ago) + + # Count events before flush + entries_before = ProductTypeEvent.objects.count() + + # Flush with retention period of 1 month flush_auditlog() - entries_after = LogEntry.objects.filter(timestamp__date__lt=datetime.now(UTC)).count() - # we have three old log entries in our testdata and added a new one - self.assertEqual(entries_before - 3 + 1, entries_after) + + # Count events after flush + entries_after = ProductTypeEvent.objects.count() + + # The old event should be deleted (2 months old > 1 month retention) + self.assertLess(entries_after, entries_before, "Old entries should be deleted") + + # Clean up + product_type.delete() diff --git a/unittests/test_import_reimport.py b/unittests/test_import_reimport.py index f51a462045a..ae7cd1286b9 100644 --- a/unittests/test_import_reimport.py +++ b/unittests/test_import_reimport.py @@ -1,7 +1,7 @@ # from unittest import skip import logging import zoneinfo -from datetime import datetime +from datetime import date, datetime from pathlib import Path from unittest.mock import patch @@ -102,12 +102,21 @@ def __init__(self, *args, **kwargs): self.anchore_grype_file_name = get_unit_tests_scans_path("anchore_grype") / "check_all_fields.json" self.anchore_grype_file_name_fix_not_available = get_unit_tests_scans_path("anchore_grype") / "fix_not_available.json" self.anchore_grype_file_name_fix_available = get_unit_tests_scans_path("anchore_grype") / "fix_available.json" + self.anchore_grype_file_name_with_ids_fabricated = get_unit_tests_scans_path("anchore_grype") / "check_all_fields_with_ids_fabricated.json" + self.anchore_grype_file_name_no_ids_fabricated = get_unit_tests_scans_path("anchore_grype") / "check_all_fields_no_ids_fabricated.json" + self.anchore_grype_file_name_different_ids_fabricated = get_unit_tests_scans_path("anchore_grype") / "check_all_fields_different_ids_fabricated.json" self.anchore_grype_scan_type = "Anchore Grype" self.checkmarx_one_open_and_false_positive = get_unit_tests_scans_path("checkmarx_one") / "one-open-one-false-positive.json" self.checkmarx_one_two_false_positive = get_unit_tests_scans_path("checkmarx_one") / "two-false-positive.json" + self.checkmarx_one_same_unique_id_3 = get_unit_tests_scans_path("checkmarx_one") / "many_findings_same_unique_id_3.json" + self.checkmarx_one_same_unique_id_4 = get_unit_tests_scans_path("checkmarx_one") / "many_findings_same_unique_id_4.json" + self.checkmarx_one_format_two = get_unit_tests_scans_path("checkmarx_one") / "checkmarx_one_format_two.json" self.scan_type_checkmarx_one = "Checkmarx One Scan" + self.bandit_large_file = get_unit_tests_scans_path("bandit") / "many_vulns.json" + self.scan_type_bandit = "Bandit Scan" + # import zap scan, testing: # - import # - active/verifed = True @@ -721,6 +730,284 @@ def test_import_veracode_reimport_veracode_different_hash_code_different_unique_ # 1 added note for the migitated finding self.assertEqual(notes_count_before + 1, self.db_notes_count()) + def test_deduplication_continues_to_next_candidate_when_regression_detected_unique_id(self): + """ + Test that deduplication continues to try the next candidate match when + the first candidate triggers a regression exception (mitigated finding). + + Real-world scenario: + 1. Import scan → finding A (active) + 2. Mitigate finding A + 3. Reimport/import again → tries to deduplicate against A (mitigated), + exception is raised, finding B is created as new active (not duplicate) + 4. Import again → should find both A (mitigated) and B (active) as candidates, + and deduplicate against B (active), not A (mitigated) + + Related to: https://github.com/DefectDojo/django-DefectDojo/issues/14010 + """ + logger.debug("testing deduplication with mitigated finding - should continue to next candidate") + + product = Product.objects.get(id=1) + + # Step 1: Import scan into engagement 1 → finding A (active) + eng1 = Engagement.objects.create( + name="Engagement 1", + product=product, + target_start=date.today(), + target_end=date.today(), + deduplication_on_engagement=False, # Product-level deduplication + ) + + with assertTestImportModelsCreated(self, imports=1, affected_findings=1, created=1): + import1 = self.import_scan_with_params( + self.checkmarx_one_format_two, + scan_type=self.scan_type_checkmarx_one, + engagement=eng1.id, + verified=True, + force_active=True, + force_verified=True, + ) + + # Get finding A + test1_id = import1["test"] + findings1 = self.get_test_findings_api(test1_id) + self.assert_finding_count_json(1, findings1) + finding1_id = findings1["results"][0]["id"] + finding1 = Finding.objects.get(id=finding1_id) + self.assertTrue(finding1.active, "Finding 1 should be active") + + # Step 2: Mitigate finding A + finding1.active = False + finding1.is_mitigated = True + finding1.mitigated = timezone.now() + finding1.out_of_scope = False + finding1.false_p = False + finding1.save(dedupe_option=False) + + # Step 3: Import again into engagement 2 → tries to deduplicate against A (mitigated), + # exception is raised, finding B is created as new active (not duplicate) + eng2 = Engagement.objects.create( + name="Engagement 2", + product=product, + target_start=date.today(), + target_end=date.today(), + deduplication_on_engagement=False, # Product-level deduplication + ) + + # The exception will be caught and logged, but finding B will be created as new + with assertTestImportModelsCreated(self, imports=1, affected_findings=1, created=1): + import2 = self.import_scan_with_params( + self.checkmarx_one_format_two, + scan_type=self.scan_type_checkmarx_one, + engagement=eng2.id, + verified=True, + force_active=True, + force_verified=True, + ) + + # Get finding B (should be active, not a duplicate because exception was raised) + test2_id = import2["test"] + findings2 = self.get_test_findings_api(test2_id) + self.assert_finding_count_json(1, findings2) + finding2_id = findings2["results"][0]["id"] + finding2 = Finding.objects.get(id=finding2_id) + self.assertTrue(finding2.active, "Finding 2 should be active") + self.assertFalse(finding2.duplicate, "Finding 2 should NOT be a duplicate (exception was raised)") + + # Step 4: Import again into engagement 3 → should find both A (mitigated) and B (active) + # as candidates, and deduplicate against B (active), not A (mitigated) + eng3 = Engagement.objects.create( + name="Engagement 3", + product=product, + target_start=date.today(), + target_end=date.today(), + deduplication_on_engagement=False, # Product-level deduplication + ) + + # The finding is created first, then deduplicated, so created=1 is expected + with assertTestImportModelsCreated(self, imports=1, affected_findings=1, created=1): + import3 = self.import_scan_with_params( + self.checkmarx_one_format_two, + scan_type=self.scan_type_checkmarx_one, + engagement=eng3.id, + verified=True, + force_active=True, + force_verified=True, + ) + + # Get finding from engagement 3 + test3_id = import3["test"] + findings3 = self.get_test_findings_api(test3_id) + self.assert_finding_count_json(1, findings3) + finding3_id = findings3["results"][0]["id"] + finding3 = Finding.objects.get(id=finding3_id) + + # Verify that finding3 is a duplicate of finding2 (active), not finding1 (mitigated) + self.assertTrue(finding3.duplicate, "Finding 3 should be a duplicate") + self.assertIsNotNone(finding3.duplicate_finding, "Finding 3 should have a duplicate_finding") + self.assertEqual( + finding3.duplicate_finding.id, + finding2.id, + "Finding 3 should be duplicate of finding 2 (active), not finding 1 (mitigated)", + ) + self.assertNotEqual( + finding3.duplicate_finding.id, + finding1.id, + "Finding 3 should NOT be duplicate of finding 1 (mitigated)", + ) + + def test_deduplication_multiple_candidates_mixed_states_unique_id(self): + """ + Test deduplication with multiple findings having the same unique_id, + where some are mitigated and some are active. The deduplication should + skip mitigated ones and use the first active one. + + Real-world scenario: + 1. Import scan → finding A (active) + 2. Mitigate finding A + 3. Import again → exception, finding B created as new active + 4. Import again → exception, finding C created as new active + 5. Mitigate finding B + 6. Import again → should find A (mitigated), B (mitigated), C (active) as candidates, + and deduplicate against C (active), not A or B (mitigated) + + Related to: https://github.com/DefectDojo/django-DefectDojo/issues/14010 + """ + logger.debug("testing deduplication with multiple candidates in mixed states") + + product = Product.objects.get(id=1) + + # Step 1: Import scan into engagement 1 → finding A (active) + eng1 = Engagement.objects.create( + name="Engagement 1", + product=product, + target_start=date.today(), + target_end=date.today(), + deduplication_on_engagement=False, + ) + + with assertTestImportModelsCreated(self, imports=1, affected_findings=1, created=1): + import1 = self.import_scan_with_params( + self.checkmarx_one_format_two, + scan_type=self.scan_type_checkmarx_one, + engagement=eng1.id, + verified=True, + force_active=True, + force_verified=True, + ) + + test1_id = import1["test"] + findings1 = self.get_test_findings_api(test1_id) + finding1_id = findings1["results"][0]["id"] + finding1 = Finding.objects.get(id=finding1_id) + + # Step 2: Mitigate finding A + finding1.active = False + finding1.is_mitigated = True + finding1.mitigated = timezone.now() + finding1.out_of_scope = False + finding1.false_p = False + finding1.save(dedupe_option=False) + + # Step 3: Import again into engagement 2 → exception, finding B created as new active + eng2 = Engagement.objects.create( + name="Engagement 2", + product=product, + target_start=date.today(), + target_end=date.today(), + deduplication_on_engagement=False, + ) + + with assertTestImportModelsCreated(self, imports=1, affected_findings=1, created=1): + import2 = self.import_scan_with_params( + self.checkmarx_one_format_two, + scan_type=self.scan_type_checkmarx_one, + engagement=eng2.id, + verified=True, + force_active=True, + force_verified=True, + ) + + test2_id = import2["test"] + findings2 = self.get_test_findings_api(test2_id) + finding2_id = findings2["results"][0]["id"] + finding2 = Finding.objects.get(id=finding2_id) + self.assertTrue(finding2.active, "Finding 2 should be active") + self.assertFalse(finding2.duplicate, "Finding 2 should NOT be a duplicate") + + # Step 4: Import again into engagement 3 → should skip finding1 (mitigated, exception), + # then deduplicate against finding2 (active), so finding C becomes a duplicate + eng3 = Engagement.objects.create( + name="Engagement 3", + product=product, + target_start=date.today(), + target_end=date.today(), + deduplication_on_engagement=False, + ) + + # The finding is created first, then deduplicated, so created=1 is expected + with assertTestImportModelsCreated(self, imports=1, affected_findings=1, created=1): + import3 = self.import_scan_with_params( + self.checkmarx_one_format_two, + scan_type=self.scan_type_checkmarx_one, + engagement=eng3.id, + verified=True, + force_active=True, + force_verified=True, + ) + + test3_id = import3["test"] + findings3 = self.get_test_findings_api(test3_id) + finding3_id = findings3["results"][0]["id"] + finding3 = Finding.objects.get(id=finding3_id) + # Finding 3 should be a duplicate of finding 2 (active), not finding 1 (mitigated) + self.assertTrue(finding3.duplicate, "Finding 3 should be a duplicate") + self.assertEqual(finding3.duplicate_finding.id, finding2.id, "Finding 3 should be duplicate of finding 2 (active)") + + # Step 5: Mitigate finding B + finding2.active = False + finding2.is_mitigated = True + finding2.mitigated = timezone.now() + finding2.out_of_scope = False + finding2.false_p = False + finding2.save(dedupe_option=False) + + # Step 6: Import again into engagement 4 → should find A (mitigated, exception), B (mitigated, exception), + # C (duplicate, skipped), so finding D is created as new active + eng4 = Engagement.objects.create( + name="Engagement 4", + product=product, + target_start=date.today(), + target_end=date.today(), + deduplication_on_engagement=False, + ) + + # The finding is created first, then deduplicated, so created=1 is expected + with assertTestImportModelsCreated(self, imports=1, affected_findings=1, created=1): + import4 = self.import_scan_with_params( + self.checkmarx_one_format_two, + scan_type=self.scan_type_checkmarx_one, + engagement=eng4.id, + verified=True, + force_active=True, + force_verified=True, + ) + + test4_id = import4["test"] + findings4 = self.get_test_findings_api(test4_id) + finding4_id = findings4["results"][0]["id"] + finding4 = Finding.objects.get(id=finding4_id) + + # Since finding3 is a duplicate (excluded from candidates), and finding1 and finding2 are mitigated (exceptions), + # finding4 should be created as new active (not a duplicate) + self.assertTrue(finding4.active, "Finding 4 should be active") + self.assertFalse(finding4.duplicate, "Finding 4 should NOT be a duplicate (all candidates raised exceptions or are duplicates)") + + # Finding4 is not a duplicate because: + # - finding1 and finding2 are mitigated (exceptions raised) + # - finding3 is a duplicate (excluded from candidates) + # So finding4 is created as new active finding + # import 0 and then reimport 1 with zap4 as extra finding, zap1 closed. # - active findings count should be 4 # - total findings count should be 5 @@ -1693,6 +1980,78 @@ def test_import_reimport_vulnerability_ids(self): self.assertEqual("GHSA-v6rh-hp5x-86rv", findings[3].vulnerability_ids[0]) self.assertEqual("CVE-2021-44420", findings[3].vulnerability_ids[1]) + def test_import_reimport_clear_vulnerability_ids(self): + """Test that vulnerability IDs are cleared when reimporting with no IDs""" + # Import scan with vulnerability IDs + import0 = self.import_scan_with_params(self.anchore_grype_file_name_with_ids_fabricated, scan_type=self.anchore_grype_scan_type) + + test_id = import0["test"] + test = Test.objects.get(id=test_id) + findings = Finding.objects.filter(test=test) + self.assertEqual(1, len(findings)) + finding = findings[0] + self.assertEqual("GHSA-v6rh-hp5x-86rv", finding.cve) + self.assertEqual(2, len(finding.vulnerability_ids)) + self.assertEqual("GHSA-v6rh-hp5x-86rv", finding.vulnerability_ids[0]) + self.assertEqual("CVE-2021-44420", finding.vulnerability_ids[1]) + + # Reimport with no vulnerability IDs - should clear existing IDs + test_type = Test_Type.objects.get(name=self.anchore_grype_scan_type) + reimport_test = Test( + engagement=test.engagement, + test_type=test_type, + scan_type=self.anchore_grype_scan_type, + target_start=datetime.now(timezone.get_current_timezone()), + target_end=datetime.now(timezone.get_current_timezone()), + ) + reimport_test.save() + + self.reimport_scan_with_params(reimport_test.id, self.anchore_grype_file_name_no_ids_fabricated, scan_type=self.anchore_grype_scan_type) + findings = Finding.objects.filter(test=reimport_test) + self.assertEqual(1, len(findings)) + finding = findings[0] + # After clearing, only the primary vuln_id should remain (GHSA-v6rh-hp5x-86rv) + # because the parser always includes the primary vulnerability.id + self.assertEqual("GHSA-v6rh-hp5x-86rv", finding.cve) + self.assertEqual(1, len(finding.vulnerability_ids)) + self.assertEqual("GHSA-v6rh-hp5x-86rv", finding.vulnerability_ids[0]) + + def test_import_reimport_change_vulnerability_ids(self): + """Test that vulnerability IDs are updated when reimporting with different IDs""" + # Import scan with initial vulnerability IDs + import0 = self.import_scan_with_params(self.anchore_grype_file_name_with_ids_fabricated, scan_type=self.anchore_grype_scan_type) + + test_id = import0["test"] + test = Test.objects.get(id=test_id) + findings = Finding.objects.filter(test=test) + self.assertEqual(1, len(findings)) + finding = findings[0] + self.assertEqual("GHSA-v6rh-hp5x-86rv", finding.cve) + self.assertEqual(2, len(finding.vulnerability_ids)) + self.assertEqual("GHSA-v6rh-hp5x-86rv", finding.vulnerability_ids[0]) + self.assertEqual("CVE-2021-44420", finding.vulnerability_ids[1]) + + # Reimport with different vulnerability IDs - should update existing IDs + test_type = Test_Type.objects.get(name=self.anchore_grype_scan_type) + reimport_test = Test( + engagement=test.engagement, + test_type=test_type, + scan_type=self.anchore_grype_scan_type, + target_start=datetime.now(timezone.get_current_timezone()), + target_end=datetime.now(timezone.get_current_timezone()), + ) + reimport_test.save() + + self.reimport_scan_with_params(reimport_test.id, self.anchore_grype_file_name_different_ids_fabricated, scan_type=self.anchore_grype_scan_type) + findings = Finding.objects.filter(test=reimport_test) + self.assertEqual(1, len(findings)) + finding = findings[0] + self.assertEqual("GHSA-v6rh-hp5x-86rv", finding.cve) + self.assertEqual(3, len(finding.vulnerability_ids)) + self.assertEqual("GHSA-v6rh-hp5x-86rv", finding.vulnerability_ids[0]) + self.assertEqual("CVE-2021-1234", finding.vulnerability_ids[1]) + self.assertEqual("CVE-2021-5678", finding.vulnerability_ids[2]) + def test_import_reimport_fix_available(self): import0 = self.import_scan_with_params(self.anchore_grype_file_name_fix_not_available, scan_type=self.anchore_grype_scan_type) test_id = import0["test"] @@ -2033,6 +2392,348 @@ def test_reimport_set_scan_date_parser_sets_date(self): date = findings["results"][0]["date"] self.assertEqual(date, "2006-12-26") + @override_settings( + IMPORT_REIMPORT_DEDUPE_BATCH_SIZE=200, + IMPORT_REIMPORT_MATCH_BATCH_SIZE=200, + ) + def test_batch_reimport_large_bandit_file(self): + """ + Test that batch reimport produces identical results to non-batch mode (which we simulate via large max batch size setting). + + Step 1: Import scan (baseline), assess active count, duplicate count + Step 2: Reimport scan (same test), assess active count, duplicate count + Step 3: Import scan in NEW engagement with batch_size=50, assess active and duplicate equal to step 1 + Step 4: Reimport scan in same new engagement (batch_size=50), assess active and duplicate equal to step 2 + """ + # Step 1: Baseline import (default batch size) + # Create engagement first and set deduplication_on_engagement before import + product_type1, _ = Product_Type.objects.get_or_create(name="PT Bandit Baseline") + product1, _ = Product.objects.get_or_create(name="P Bandit Baseline", prod_type=product_type1) + engagement1 = Engagement.objects.create( + name="E Bandit Baseline", + product=product1, + target_start=timezone.now(), + target_end=timezone.now(), + ) + engagement1.deduplication_on_engagement = True + engagement1.save() + engagement1_id = engagement1.id + + import1 = self.import_scan_with_params( + self.bandit_large_file, + scan_type=self.scan_type_bandit, + engagement=engagement1_id, + product_type_name=None, + product_name=None, + engagement_name=None, + auto_create_context=False, + ) + test1_id = import1["test"] + + # Assess step 1: active count, duplicate count + step1_active = Finding.objects.filter( + test__engagement_id=engagement1_id, + active=True, + duplicate=False, + ).count() + step1_duplicate = Finding.objects.filter( + test__engagement_id=engagement1_id, + duplicate=True, + ).count() + + # Assert step 1 specific counts + # Each assertion is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + with self.subTest(step=1, metric="active"): + self.assertEqual(step1_active, 213, "Step 1 active count") + with self.subTest(step=1, metric="duplicate"): + self.assertEqual(step1_duplicate, 1, "Step 1 duplicate count") + + # Step 2: Reimport scan (same test) + self.reimport_scan_with_params( + test1_id, + self.bandit_large_file, + scan_type=self.scan_type_bandit, + ) + + # Assess step 2: active count, duplicate count + step2_active = Finding.objects.filter( + test__engagement_id=engagement1_id, + active=True, + duplicate=False, + ).count() + step2_duplicate = Finding.objects.filter( + test__engagement_id=engagement1_id, + duplicate=True, + ).count() + + # Assert step 2 specific counts + # Each assertion is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + with self.subTest(step=2, metric="active"): + self.assertEqual(step2_active, 213, "Step 2 active count") + with self.subTest(step=2, metric="duplicate"): + self.assertEqual(step2_duplicate, 1, "Step 2 duplicate count") + + # Step 3: Import scan in NEW engagement with batch_size=50 + with override_settings( + IMPORT_REIMPORT_DEDUPE_BATCH_SIZE=50, + IMPORT_REIMPORT_MATCH_BATCH_SIZE=50, + ): + # Create engagement first and set deduplication_on_engagement before import + product_type2, _ = Product_Type.objects.get_or_create(name="PT Bandit Batch") + product2, _ = Product.objects.get_or_create(name="P Bandit Batch", prod_type=product_type2) + engagement2 = Engagement.objects.create( + name="E Bandit Batch", + product=product2, + target_start=timezone.now(), + target_end=timezone.now(), + ) + engagement2.deduplication_on_engagement = True + engagement2.save() + engagement2_id = engagement2.id + + import2 = self.import_scan_with_params( + self.bandit_large_file, + scan_type=self.scan_type_bandit, + engagement=engagement2_id, + product_type_name=None, + product_name=None, + engagement_name=None, + auto_create_context=False, + ) + test2_id = import2["test"] + + # Assess step 3: active and duplicate should equal step 1 + step3_active = Finding.objects.filter( + test__engagement_id=engagement2_id, + active=True, + duplicate=False, + ).count() + step3_duplicate = Finding.objects.filter( + test__engagement_id=engagement2_id, + duplicate=True, + ).count() + + # Each assertion is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + with self.subTest(step=3, metric="active"): + self.assertEqual( + step3_active, + step1_active, + "Step 3 active count should equal step 1 (baseline import)", + ) + with self.subTest(step=3, metric="duplicate"): + self.assertEqual( + step3_duplicate, + step1_duplicate, + "Step 3 duplicate count should equal step 1 (baseline import)", + ) + + # Step 4: Reimport scan in same new engagement (batch_size=50) + self.reimport_scan_with_params( + test2_id, + self.bandit_large_file, + scan_type=self.scan_type_bandit, + ) + + # Assess step 4: active and duplicate should equal step 2 + step4_active = Finding.objects.filter( + test__engagement_id=engagement2_id, + active=True, + duplicate=False, + ).count() + step4_duplicate = Finding.objects.filter( + test__engagement_id=engagement2_id, + duplicate=True, + ).count() + + # Each assertion is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + with self.subTest(step=4, metric="active"): + self.assertEqual( + step4_active, + step2_active, + "Step 4 active count should equal step 2 (baseline reimport)", + ) + with self.subTest(step=4, metric="duplicate"): + self.assertEqual( + step4_duplicate, + step2_duplicate, + "Step 4 duplicate count should equal step 2 (baseline reimport)", + ) + + def test_batch_deduplication_large_bandit_file(self): + """ + Test that batch deduplication produces identical results to non-batch mode (which we simulate via large max batch size setting). + + Step 1: Import scan (baseline), assess active count, duplicate count + Step 2: Import scan again (same engagement), assess active count, duplicate count + Step 3: Import scan in NEW engagement with batch_size=50, assess active and duplicate equal to step 1 + Step 4: Import scan again in same new engagement (batch_size=50), assess active and duplicate equal to step 2 + """ + # Step 1: Baseline import (default batch size) + # Create engagement first and set deduplication_on_engagement before import + product_type1, _ = Product_Type.objects.get_or_create(name="PT Bandit Baseline Dedupe") + product1, _ = Product.objects.get_or_create(name="P Bandit Baseline Dedupe", prod_type=product_type1) + engagement1 = Engagement.objects.create( + name="E Bandit Baseline Dedupe", + product=product1, + target_start=timezone.now(), + target_end=timezone.now(), + ) + engagement1.deduplication_on_engagement = True + engagement1.save() + engagement1_id = engagement1.id + + self.import_scan_with_params( + self.bandit_large_file, + scan_type=self.scan_type_bandit, + engagement=engagement1_id, + product_type_name=None, + product_name=None, + engagement_name=None, + auto_create_context=False, + ) + + # Assess step 1: active count, duplicate count + step1_active = Finding.objects.filter( + test__engagement_id=engagement1_id, + active=True, + duplicate=False, + ).count() + step1_duplicate = Finding.objects.filter( + test__engagement_id=engagement1_id, + duplicate=True, + ).count() + + # Assert step 1 specific counts + self.assertEqual(step1_active, 213, "Step 1 active count") + self.assertEqual(step1_duplicate, 1, "Step 1 duplicate count") + + # Step 2: Import scan again (same engagement) + self.import_scan_with_params( + self.bandit_large_file, + scan_type=self.scan_type_bandit, + engagement=engagement1_id, + product_type_name=None, + product_name=None, + engagement_name=None, + auto_create_context=False, + ) + + # Assess step 2: active count, duplicate count + step2_active = Finding.objects.filter( + test__engagement_id=engagement1_id, + active=True, + duplicate=False, + ).count() + step2_duplicate = Finding.objects.filter( + test__engagement_id=engagement1_id, + duplicate=True, + ).count() + + # Assert step 2 specific counts + self.assertEqual(step2_active, 213, "Step 2 active count") + self.assertEqual(step2_duplicate, 215, "Step 2 duplicate count") + + # Step 3: Import scan in NEW engagement with batch_size=50 + with override_settings( + IMPORT_REIMPORT_DEDUPE_BATCH_SIZE=50, + IMPORT_REIMPORT_MATCH_BATCH_SIZE=50, + ): + # Create engagement first and set deduplication_on_engagement before import + product_type2, _ = Product_Type.objects.get_or_create(name="PT Bandit Batch Dedupe") + product2, _ = Product.objects.get_or_create(name="P Bandit Batch Dedupe", prod_type=product_type2) + engagement2 = Engagement.objects.create( + name="E Bandit Batch Dedupe", + product=product2, + target_start=timezone.now(), + target_end=timezone.now(), + ) + engagement2.deduplication_on_engagement = True + engagement2.save() + engagement2_id = engagement2.id + + self.import_scan_with_params( + self.bandit_large_file, + scan_type=self.scan_type_bandit, + engagement=engagement2_id, + product_type_name=None, + product_name=None, + engagement_name=None, + auto_create_context=False, + ) + + # Assess step 3: active and duplicate should equal step 1 + step3_active = Finding.objects.filter( + test__engagement_id=engagement2_id, + active=True, + duplicate=False, + ).count() + step3_duplicate = Finding.objects.filter( + test__engagement_id=engagement2_id, + duplicate=True, + ).count() + + # Each assertion is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + with self.subTest(step=3, metric="active"): + self.assertEqual( + step3_active, + step1_active, + "Step 3 active count should equal step 1 (baseline import)", + ) + with self.subTest(step=3, metric="duplicate"): + self.assertEqual( + step3_duplicate, + step1_duplicate, + "Step 3 duplicate count should equal step 1 (baseline import)", + ) + + # Step 4: Import scan again in same new engagement (batch_size=50) + self.import_scan_with_params( + self.bandit_large_file, + scan_type=self.scan_type_bandit, + engagement=engagement2_id, + product_type_name=None, + product_name=None, + engagement_name=None, + auto_create_context=False, + ) + + # Assess step 4: active and duplicate should equal step 2 + step4_active = Finding.objects.filter( + test__engagement_id=engagement2_id, + active=True, + duplicate=False, + ).count() + step4_duplicate = Finding.objects.filter( + test__engagement_id=engagement2_id, + duplicate=True, + ).count() + + # Each assertion is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + with self.subTest(step=4, metric="active"): + self.assertEqual( + step4_active, + step2_active, + "Step 4 active count should equal step 2 (baseline second import)", + ) + with self.subTest(step=4, metric="duplicate"): + self.assertEqual( + step4_duplicate, + step2_duplicate, + "Step 4 duplicate count should equal step 2 (baseline second import)", + ) + @override_settings( IMPORT_REIMPORT_DEDUPE_BATCH_SIZE=50, IMPORT_REIMPORT_MATCH_BATCH_SIZE=50, diff --git a/unittests/test_importers_importer.py b/unittests/test_importers_importer.py index 4077648c812..a4eec5dd737 100644 --- a/unittests/test_importers_importer.py +++ b/unittests/test_importers_importer.py @@ -9,7 +9,16 @@ from dojo.importers.default_importer import DefaultImporter from dojo.importers.default_reimporter import DefaultReImporter -from dojo.models import Development_Environment, Engagement, Finding, Product, Product_Type, Test, User +from dojo.models import ( + Development_Environment, + Engagement, + Finding, + Product, + Product_Type, + Test, + User, + Vulnerability_Id, +) from dojo.tools.gitlab_sast.parser import GitlabSastParser from dojo.tools.sarif.parser import SarifParser from dojo.utils import get_object_or_none @@ -786,8 +795,7 @@ def create_default_data(self): "scan_type": NPM_AUDIT_SCAN_TYPE, } - @patch("dojo.importers.base_importer.Vulnerability_Id", autospec=True) - def test_handle_vulnerability_ids_references_and_cve(self, mock): + def test_handle_vulnerability_ids_references_and_cve(self): # Why doesn't this test use the test db and query for one? vulnerability_ids = ["CVE", "REF-1", "REF-2"] finding = Finding() @@ -795,7 +803,7 @@ def test_handle_vulnerability_ids_references_and_cve(self, mock): finding.test = self.test finding.reporter = self.testuser finding.save() - DefaultImporter(**self.importer_data).process_vulnerability_ids(finding) + DefaultImporter(**self.importer_data).store_vulnerability_ids(finding) self.assertEqual("CVE", finding.vulnerability_ids[0]) self.assertEqual("CVE", finding.cve) @@ -804,8 +812,7 @@ def test_handle_vulnerability_ids_references_and_cve(self, mock): self.assertEqual("REF-2", finding.vulnerability_ids[2]) finding.delete() - @patch("dojo.importers.base_importer.Vulnerability_Id", autospec=True) - def test_handle_no_vulnerability_ids_references_and_cve(self, mock): + def test_handle_no_vulnerability_ids_references_and_cve(self): vulnerability_ids = ["CVE"] finding = Finding() finding.test = self.test @@ -813,22 +820,21 @@ def test_handle_no_vulnerability_ids_references_and_cve(self, mock): finding.save() finding.unsaved_vulnerability_ids = vulnerability_ids - DefaultImporter(**self.importer_data).process_vulnerability_ids(finding) + DefaultImporter(**self.importer_data).store_vulnerability_ids(finding) self.assertEqual("CVE", finding.vulnerability_ids[0]) self.assertEqual("CVE", finding.cve) self.assertEqual(vulnerability_ids, finding.unsaved_vulnerability_ids) finding.delete() - @patch("dojo.importers.base_importer.Vulnerability_Id", autospec=True) - def test_handle_vulnerability_ids_references_and_no_cve(self, mock): + def test_handle_vulnerability_ids_references_and_no_cve(self): vulnerability_ids = ["REF-1", "REF-2"] finding = Finding() finding.test = self.test finding.reporter = self.testuser finding.save() finding.unsaved_vulnerability_ids = vulnerability_ids - DefaultImporter(**self.importer_data).process_vulnerability_ids(finding) + DefaultImporter(**self.importer_data).store_vulnerability_ids(finding) self.assertEqual("REF-1", finding.vulnerability_ids[0]) self.assertEqual("REF-1", finding.cve) @@ -836,14 +842,87 @@ def test_handle_vulnerability_ids_references_and_no_cve(self, mock): self.assertEqual("REF-2", finding.vulnerability_ids[1]) finding.delete() - @patch("dojo.importers.base_importer.Vulnerability_Id", autospec=True) - def test_no_handle_vulnerability_ids_references_and_no_cve(self, mock): + def test_no_handle_vulnerability_ids_references_and_no_cve(self): finding = Finding() finding.test = self.test finding.reporter = self.testuser finding.save() - DefaultImporter(**self.importer_data).process_vulnerability_ids(finding) + DefaultImporter(**self.importer_data).store_vulnerability_ids(finding) self.assertEqual(finding.cve, None) self.assertEqual(finding.unsaved_vulnerability_ids, None) self.assertEqual(finding.vulnerability_ids, []) finding.delete() + + def test_clear_vulnerability_ids_on_empty_list(self): + """Test that vulnerability IDs are cleared when an empty list is provided""" + # Create a finding with existing vulnerability IDs + finding = Finding() + finding.test = self.test + finding.reporter = self.testuser + finding.save() + + # Add some vulnerability IDs + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-1234") + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-5678") + finding.cve = "CVE-2020-1234" + finding.save() + + # Verify initial state + self.assertEqual(2, len(finding.vulnerability_ids)) + self.assertEqual("CVE-2020-1234", finding.cve) + + # Process with empty list - should clear all IDs + finding.unsaved_vulnerability_ids = [] + DefaultReImporter(test=self.test, environment=self.importer_data["environment"], scan_type=self.importer_data["scan_type"]).reconcile_vulnerability_ids(finding) + # Save the finding to persist the cve=None change + finding.save() + + # Get fresh finding from database to avoid cached property issues + finding = Finding.objects.get(pk=finding.pk) + + # Verify IDs are cleared + self.assertEqual(0, len(finding.vulnerability_ids)) + self.assertEqual(None, finding.cve) + # Verify no Vulnerability_Id objects exist for this finding + self.assertEqual(0, Vulnerability_Id.objects.filter(finding=finding).count()) + finding.delete() + + def test_change_vulnerability_ids_on_reimport(self): + """Test that vulnerability IDs are updated when different IDs are provided""" + # Create a finding with existing vulnerability IDs + finding = Finding() + finding.test = self.test + finding.reporter = self.testuser + finding.save() + + # Add initial vulnerability IDs + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-1234") + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-5678") + finding.cve = "CVE-2020-1234" + finding.save() + + # Verify initial state + self.assertEqual(2, len(finding.vulnerability_ids)) + self.assertEqual("CVE-2020-1234", finding.vulnerability_ids[0]) + self.assertEqual("CVE-2020-5678", finding.vulnerability_ids[1]) + self.assertEqual("CVE-2020-1234", finding.cve) + + # Process with different IDs - should replace old IDs + new_vulnerability_ids = ["CVE-2021-9999", "GHSA-xxxx-yyyy"] + finding.unsaved_vulnerability_ids = new_vulnerability_ids + DefaultReImporter(test=self.test, environment=self.importer_data["environment"], scan_type=self.importer_data["scan_type"]).reconcile_vulnerability_ids(finding) + # Save the finding to persist the cve change + finding.save() + + # Get fresh finding from database to avoid cached property issues + finding = Finding.objects.get(pk=finding.pk) + + # Verify old IDs are removed and new IDs are present + self.assertEqual(2, len(finding.vulnerability_ids)) + self.assertEqual("CVE-2021-9999", finding.vulnerability_ids[0]) + self.assertEqual("GHSA-xxxx-yyyy", finding.vulnerability_ids[1]) + self.assertEqual("CVE-2021-9999", finding.cve) + # Verify only new Vulnerability_Id objects exist + vuln_ids = list(Vulnerability_Id.objects.filter(finding=finding).values_list("vulnerability_id", flat=True)) + self.assertEqual(set(new_vulnerability_ids), set(vuln_ids)) + finding.delete() diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 83058b29f10..1e7b05d8fe5 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -1,3 +1,23 @@ +""" +Performance tests for importers. + +These tests verify that import and reimport operations maintain acceptable query counts +and async task counts to prevent performance regressions. + +Counts can be updated via the Python script at scripts/update_performance_test_counts.py. +However, counts must be verified to ensure no implicit performance regressions are introduced. +When counts change, review the differences carefully to determine if they represent: +- Legitimate optimizations (counts decreasing) +- Acceptable changes due to feature additions (counts increasing with justification) +- Unintended performance regressions (counts increasing without clear reason) + +Always verify updated counts by: +1. Running the update script to see the differences +2. Reviewing the changes to understand why counts changed +3. Running the verification step to ensure all tests pass +4. Investigating any unexpected increases in query or task counts +""" + import logging from contextlib import contextmanager @@ -33,7 +53,9 @@ STACK_HAWK_SCAN_TYPE = "StackHawk HawkScan" -class TestDojoImporterPerformance(DojoTestCase): +class TestDojoImporterPerformanceBase(DojoTestCase): + + """Base class for performance tests with shared setup and helper methods.""" def setUp(self): super().setUp() @@ -77,113 +99,163 @@ def _assertNumAsyncTask(self, num): ) logger.debug(msg) - def _import_reimport_performance(self, expected_num_queries1, expected_num_async_tasks1, expected_num_queries2, expected_num_async_tasks2, expected_num_queries3, expected_num_async_tasks3): - """ - Log output can be quite large as when the assertNumQueries fails, all queries are printed. - It could be usefule to capture the output in `less`: - ./run-unittest.sh --test-case unittests.test_importers_performance.TestDojoImporterPerformance 2>&1 | less - Then search for `expected` to find the lines where the expected number of queries is printed. - Or you can use `grep` to filter the output: - ./run-unittest.sh --test-case unittests.test_importers_performance.TestDojoImporterPerformance 2>&1 | grep expected -B 10 - """ + def _create_test_objects(self, product_name, engagement_name): + """Helper method to create test product, engagement, lead user, and environment.""" product_type, _created = Product_Type.objects.get_or_create(name="test") product, _created = Product.objects.get_or_create( - name="TestDojoDefaultImporter", + name=product_name, prod_type=product_type, ) engagement, _created = Engagement.objects.get_or_create( - name="Test Create Engagement", + name=engagement_name, product=product, target_start=timezone.now(), target_end=timezone.now(), ) lead, _ = User.objects.get_or_create(username="admin") environment, _ = Development_Environment.objects.get_or_create(name="Development") + return product, engagement, lead, environment + + def _import_reimport_performance( + self, + expected_num_queries1, + expected_num_async_tasks1, + expected_num_queries2, + expected_num_async_tasks2, + expected_num_queries3, + expected_num_async_tasks3, + scan_file1, + scan_file2, + scan_file3, + scan_type, + product_name, + engagement_name, + ): + """ + Test import/reimport/reimport performance with specified scan files and scan type. + Log output can be quite large as when the assertNumQueries fails, all queries are printed. + """ + _, engagement, lead, environment = self._create_test_objects( + product_name, + engagement_name, + ) - # first import the subset which missed one finding and a couple of endpoints on some of the findings - with ( + # First import + # Each assertion context manager is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + # Nested with statements are intentional - each assertion needs its own subTest wrapper. + with ( # noqa: SIM117 self.subTest("import1"), impersonate(Dojo_User.objects.get(username="admin")), - self.assertNumQueries(expected_num_queries1), - self._assertNumAsyncTask(expected_num_async_tasks1), - STACK_HAWK_SUBSET_FILENAME.open(encoding="utf-8") as scan, + scan_file1.open(encoding="utf-8") as scan, ): - import_options = { - "user": lead, - "lead": lead, - "scan_date": None, - "environment": environment, - "minimum_severity": "Info", - "active": True, - "verified": True, - "sync": True, - "scan_type": STACK_HAWK_SCAN_TYPE, - "engagement": engagement, - "tags": ["performance-test", "tag-in-param", "go-faster"], - "apply_tags_to_findings": True, - } - importer = DefaultImporter(**import_options) - test, _, _len_new_findings, _len_closed_findings, _, _, _ = importer.process_scan(scan) - - # use reimport with the full report so it add a finding and some endpoints - with ( + with self.subTest(step="import1", metric="queries"): + with self.assertNumQueries(expected_num_queries1): + with self.subTest(step="import1", metric="async_tasks"): + with self._assertNumAsyncTask(expected_num_async_tasks1): + import_options = { + "user": lead, + "lead": lead, + "scan_date": None, + "environment": environment, + "minimum_severity": "Info", + "active": True, + "verified": True, + "sync": True, + "scan_type": scan_type, + "engagement": engagement, + "tags": ["performance-test", "tag-in-param", "go-faster"], + "apply_tags_to_findings": True, + } + importer = DefaultImporter(**import_options) + test, _, _len_new_findings, _len_closed_findings, _, _, _ = importer.process_scan(scan) + + # Second import (reimport) + # Each assertion context manager is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + # Nested with statements are intentional - each assertion needs its own subTest wrapper. + with ( # noqa: SIM117 self.subTest("reimport1"), impersonate(Dojo_User.objects.get(username="admin")), - self.assertNumQueries(expected_num_queries2), - self._assertNumAsyncTask(expected_num_async_tasks2), - STACK_HAWK_FILENAME.open(encoding="utf-8") as scan, + scan_file2.open(encoding="utf-8") as scan, ): - reimport_options = { - "test": test, - "user": lead, - "lead": lead, - "scan_date": None, - "minimum_severity": "Info", - "active": True, - "verified": True, - "sync": True, - "scan_type": STACK_HAWK_SCAN_TYPE, - "tags": ["performance-test-reimport", "reimport-tag-in-param", "reimport-go-faster"], - "apply_tags_to_findings": True, - } - reimporter = DefaultReImporter(**reimport_options) - test, _, _len_new_findings, _len_closed_findings, _, _, _ = reimporter.process_scan(scan) - - # use reimport with the subset again to close a finding and mitigate some endpoints - with ( + with self.subTest(step="reimport1", metric="queries"): + with self.assertNumQueries(expected_num_queries2): + with self.subTest(step="reimport1", metric="async_tasks"): + with self._assertNumAsyncTask(expected_num_async_tasks2): + reimport_options = { + "test": test, + "user": lead, + "lead": lead, + "scan_date": None, + "minimum_severity": "Info", + "active": True, + "verified": True, + "sync": True, + "scan_type": scan_type, + "tags": ["performance-test-reimport", "reimport-tag-in-param", "reimport-go-faster"], + "apply_tags_to_findings": True, + } + reimporter = DefaultReImporter(**reimport_options) + test, _, _len_new_findings, _len_closed_findings, _, _, _ = reimporter.process_scan(scan) + + # Third import (reimport again) + # Each assertion context manager is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + # Nested with statements are intentional - each assertion needs its own subTest wrapper. + with ( # noqa: SIM117 self.subTest("reimport2"), impersonate(Dojo_User.objects.get(username="admin")), - self.assertNumQueries(expected_num_queries3), - self._assertNumAsyncTask(expected_num_async_tasks3), - STACK_HAWK_SUBSET_FILENAME.open(encoding="utf-8") as scan, + scan_file3.open(encoding="utf-8") as scan, ): - reimport_options = { - "test": test, - "user": lead, - "lead": lead, - "scan_date": None, - "minimum_severity": "Info", - "active": True, - "verified": True, - "sync": True, - "scan_type": STACK_HAWK_SCAN_TYPE, - } - reimporter = DefaultReImporter(**reimport_options) - test, _, _len_new_findings, _len_closed_findings, _, _, _ = reimporter.process_scan(scan) - - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-auditlog") - def test_import_reimport_reimport_performance_async(self): - # Ensure django-auditlog is properly configured for this test - configure_audit_system() - configure_pghistory_triggers() + with self.subTest(step="reimport2", metric="queries"): + with self.assertNumQueries(expected_num_queries3): + with self.subTest(step="reimport2", metric="async_tasks"): + with self._assertNumAsyncTask(expected_num_async_tasks3): + reimport_options = { + "test": test, + "user": lead, + "lead": lead, + "scan_date": None, + "minimum_severity": "Info", + "active": True, + "verified": True, + "sync": True, + "scan_type": scan_type, + } + reimporter = DefaultReImporter(**reimport_options) + test, _, _len_new_findings, _len_closed_findings, _, _, _ = reimporter.process_scan(scan) + + +class TestDojoImporterPerformanceSmall(TestDojoImporterPerformanceBase): + + """Performance tests using small sample files (StackHawk, ~6 findings).""" - self._import_reimport_performance( - expected_num_queries1=340, - expected_num_async_tasks1=7, - expected_num_queries2=288, - expected_num_async_tasks2=18, - expected_num_queries3=175, - expected_num_async_tasks3=17, + def _import_reimport_performance(self, expected_num_queries1, expected_num_async_tasks1, expected_num_queries2, expected_num_async_tasks2, expected_num_queries3, expected_num_async_tasks3): + """ + Log output can be quite large as when the assertNumQueries fails, all queries are printed. + It could be usefule to capture the output in `less`: + ./run-unittest.sh --test-case unittests.test_importers_performance.TestDojoImporterPerformanceSmall 2>&1 | less + Then search for `expected` to find the lines where the expected number of queries is printed. + Or you can use `grep` to filter the output: + ./run-unittest.sh --test-case unittests.test_importers_performance.TestDojoImporterPerformanceSmall 2>&1 | grep expected -B 10 + """ + return super()._import_reimport_performance( + expected_num_queries1, + expected_num_async_tasks1, + expected_num_queries2, + expected_num_async_tasks2, + expected_num_queries3, + expected_num_async_tasks3, + scan_file1=STACK_HAWK_SUBSET_FILENAME, + scan_file2=STACK_HAWK_FILENAME, + scan_file3=STACK_HAWK_SUBSET_FILENAME, + scan_type=STACK_HAWK_SCAN_TYPE, + product_name="TestDojoDefaultImporter", + engagement_name="Test Create Engagement", ) - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-pghistory") + @override_settings(ENABLE_AUDITLOG=True) def test_import_reimport_reimport_performance_pghistory_async(self): """ This test checks the performance of the importers when using django-pghistory with async enabled. @@ -195,37 +267,13 @@ def test_import_reimport_reimport_performance_pghistory_async(self): self._import_reimport_performance( expected_num_queries1=306, expected_num_async_tasks1=7, - expected_num_queries2=281, + expected_num_queries2=232, expected_num_async_tasks2=18, - expected_num_queries3=170, + expected_num_queries3=114, expected_num_async_tasks3=17, ) - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-auditlog") - def test_import_reimport_reimport_performance_no_async(self): - """ - This test checks the performance of the importers when they are run in sync mode. - The reason for this is that we also want to be aware of when a PR affects the number of queries - or async tasks created by a background task. - The impersonate context manager above does not work as expected for disabling async, - so we patch the we_want_async decorator to always return False. - """ - configure_audit_system() - configure_pghistory_triggers() - - testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True - testuser.usercontactinfo.save() - self._import_reimport_performance( - expected_num_queries1=346, - expected_num_async_tasks1=6, - expected_num_queries2=294, - expected_num_async_tasks2=17, - expected_num_queries3=181, - expected_num_async_tasks3=16, - ) - - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-pghistory") + @override_settings(ENABLE_AUDITLOG=True) def test_import_reimport_reimport_performance_pghistory_no_async(self): """ This test checks the performance of the importers when using django-pghistory with async disabled. @@ -239,41 +287,15 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=312, + expected_num_queries1=313, expected_num_async_tasks1=6, - expected_num_queries2=287, + expected_num_queries2=239, expected_num_async_tasks2=17, - expected_num_queries3=176, + expected_num_queries3=121, expected_num_async_tasks3=16, ) - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-auditlog") - def test_import_reimport_reimport_performance_no_async_with_product_grading(self): - """ - This test checks the performance of the importers when they are run in sync mode. - The reason for this is that we also want to be aware of when a PR affects the number of queries - or async tasks created by a background task. - The impersonate context manager above does not work as expected for disabling async, - so we patch the we_want_async decorator to always return False. - """ - configure_audit_system() - configure_pghistory_triggers() - - testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True - testuser.usercontactinfo.save() - self.system_settings(enable_product_grade=True) - - self._import_reimport_performance( - expected_num_queries1=348, - expected_num_async_tasks1=8, - expected_num_queries2=296, - expected_num_async_tasks2=19, - expected_num_queries3=183, - expected_num_async_tasks3=18, - ) - - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-pghistory") + @override_settings(ENABLE_AUDITLOG=True) def test_import_reimport_reimport_performance_pghistory_no_async_with_product_grading(self): """ This test checks the performance of the importers when using django-pghistory with async disabled and product grading enabled. @@ -288,11 +310,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=314, + expected_num_queries1=315, expected_num_async_tasks1=8, - expected_num_queries2=289, + expected_num_queries2=241, expected_num_async_tasks2=19, - expected_num_queries3=178, + expected_num_queries3=123, expected_num_async_tasks3=18, ) @@ -303,61 +325,64 @@ def _deduplication_performance(self, expected_num_queries1, expected_num_async_t The second import should result in all findings being marked as duplicates. This is different from reimport as we create a new test each time. """ - product_type, _created = Product_Type.objects.get_or_create(name="test") - product, _created = Product.objects.get_or_create( - name="TestDojoDeduplicationPerformance", - prod_type=product_type, - ) - engagement, _created = Engagement.objects.get_or_create( - name="Test Deduplication Performance Engagement", - product=product, - target_start=timezone.now(), - target_end=timezone.now(), + _, engagement, lead, environment = self._create_test_objects( + "TestDojoDeduplicationPerformance", + "Test Deduplication Performance Engagement", ) - lead, _ = User.objects.get_or_create(username="admin") - environment, _ = Development_Environment.objects.get_or_create(name="Development") # First import - all findings should be new - with ( + # Each assertion context manager is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + # Nested with statements are intentional - each assertion needs its own subTest wrapper. + with ( # noqa: SIM117 self.subTest("first_import"), impersonate(Dojo_User.objects.get(username="admin")), - self.assertNumQueries(expected_num_queries1), - self._assertNumAsyncTask(expected_num_async_tasks1), STACK_HAWK_FILENAME.open(encoding="utf-8") as scan, ): - import_options = { - "user": lead, - "lead": lead, - "scan_date": None, - "environment": environment, - "minimum_severity": "Info", - "active": True, - "verified": True, - "scan_type": STACK_HAWK_SCAN_TYPE, - "engagement": engagement, - } - importer = DefaultImporter(**import_options) - _, _, len_new_findings1, len_closed_findings1, _, _, _ = importer.process_scan(scan) + with self.subTest(step="first_import", metric="queries"): + with self.assertNumQueries(expected_num_queries1): + with self.subTest(step="first_import", metric="async_tasks"): + with self._assertNumAsyncTask(expected_num_async_tasks1): + import_options = { + "user": lead, + "lead": lead, + "scan_date": None, + "environment": environment, + "minimum_severity": "Info", + "active": True, + "verified": True, + "scan_type": STACK_HAWK_SCAN_TYPE, + "engagement": engagement, + } + importer = DefaultImporter(**import_options) + _, _, len_new_findings1, len_closed_findings1, _, _, _ = importer.process_scan(scan) # Second import - all findings should be duplicates - with ( + # Each assertion context manager is wrapped in its own subTest so that if one fails, the others still run. + # This allows us to see all count mismatches in a single test run, making it easier to fix + # all incorrect expected values at once rather than fixing them one at a time. + # Nested with statements are intentional - each assertion needs its own subTest wrapper. + with ( # noqa: SIM117 self.subTest("second_import"), impersonate(Dojo_User.objects.get(username="admin")), - self.assertNumQueries(expected_num_queries2), - self._assertNumAsyncTask(expected_num_async_tasks2), STACK_HAWK_FILENAME.open(encoding="utf-8") as scan, ): - import_options = { - "user": lead, - "lead": lead, - "scan_date": None, - "environment": environment, - "minimum_severity": "Info", - "active": True, - "verified": True, - "scan_type": STACK_HAWK_SCAN_TYPE, - "engagement": engagement, - } - importer = DefaultImporter(**import_options) - _, _, len_new_findings2, len_closed_findings2, _, _, _ = importer.process_scan(scan) + with self.subTest(step="second_import", metric="queries"): + with self.assertNumQueries(expected_num_queries2): + with self.subTest(step="second_import", metric="async_tasks"): + with self._assertNumAsyncTask(expected_num_async_tasks2): + import_options = { + "user": lead, + "lead": lead, + "scan_date": None, + "environment": environment, + "minimum_severity": "Info", + "active": True, + "verified": True, + "scan_type": STACK_HAWK_SCAN_TYPE, + "engagement": engagement, + } + importer = DefaultImporter(**import_options) + _, _, len_new_findings2, len_closed_findings2, _, _, _ = importer.process_scan(scan) # Log the results for analysis logger.debug(f"First import: {len_new_findings1} new findings, {len_closed_findings1} closed findings") @@ -398,27 +423,7 @@ def _deduplication_performance(self, expected_num_queries1, expected_num_async_t total_findings = Finding.objects.filter(test__engagement=engagement).count() self.assertEqual(total_findings, 12, f"Expected 12 total findings, got {total_findings}") - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-auditlog") - def test_deduplication_performance_async(self): - """ - Test deduplication performance with async tasks enabled. - This test imports the same scan twice to measure deduplication query and task overhead. - """ - configure_audit_system() - configure_pghistory_triggers() - - # Enable deduplication - self.system_settings(enable_deduplication=True) - - self._deduplication_performance( - expected_num_queries1=311, - expected_num_async_tasks1=8, - expected_num_queries2=204, - expected_num_async_tasks2=8, - check_duplicates=False, # Async mode - deduplication happens later - ) - - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-pghistory") + @override_settings(ENABLE_AUDITLOG=True) def test_deduplication_performance_pghistory_async(self): """Test deduplication performance with django-pghistory and async tasks enabled.""" configure_audit_system() @@ -435,27 +440,7 @@ def test_deduplication_performance_pghistory_async(self): check_duplicates=False, # Async mode - deduplication happens later ) - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-auditlog") - def test_deduplication_performance_no_async(self): - """Test deduplication performance with async tasks disabled.""" - configure_audit_system() - configure_pghistory_triggers() - - # Enable deduplication - self.system_settings(enable_deduplication=True) - - testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True - testuser.usercontactinfo.save() - - self._deduplication_performance( - expected_num_queries1=317, - expected_num_async_tasks1=7, - expected_num_queries2=282, - expected_num_async_tasks2=7, - ) - - @override_settings(ENABLE_AUDITLOG=True, AUDITLOG_TYPE="django-pghistory") + @override_settings(ENABLE_AUDITLOG=True) def test_deduplication_performance_pghistory_no_async(self): """Test deduplication performance with django-pghistory and async tasks disabled.""" configure_audit_system() @@ -469,8 +454,9 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=281, + expected_num_queries1=282, expected_num_async_tasks1=7, - expected_num_queries2=245, + expected_num_queries2=246, expected_num_async_tasks2=7, + ) diff --git a/unittests/test_test_type_active_toggle.py b/unittests/test_test_type_active_toggle.py new file mode 100644 index 00000000000..1d0e8a55644 --- /dev/null +++ b/unittests/test_test_type_active_toggle.py @@ -0,0 +1,26 @@ + +from django.test import TestCase + +from dojo.filters import FindingFilter +from dojo.models import Test_Type +from dojo.utils import get_visible_scan_types + + +class TestFindingFilterActiveInactiveTestTypes(TestCase): + def setUp(self): + self.active_type = Test_Type.objects.create(name="Nessus Scan", active=True) + self.inactive_type = Test_Type.objects.create(name="Burp Scan", active=False) + + def test_only_active_types_in_filter(self): + filter_instance = FindingFilter(data={}) + self.assertIn("test__test_type", filter_instance.form.fields) + queryset = filter_instance.form.fields["test__test_type"].queryset + actual_names = set(queryset.values_list("name", flat=True)) + self.assertIn(self.active_type.name, actual_names) + self.assertNotIn(self.inactive_type.name, actual_names) + + def test_helper_function_returns_only_active(self): + visible = get_visible_scan_types() + names = set(visible.values_list("name", flat=True)) + self.assertIn(self.active_type.name, names) + self.assertNotIn(self.inactive_type.name, names) diff --git a/unittests/test_user_ui_timestamps.py b/unittests/test_user_ui_timestamps.py new file mode 100644 index 00000000000..367619466db --- /dev/null +++ b/unittests/test_user_ui_timestamps.py @@ -0,0 +1,92 @@ +from datetime import UTC, datetime +from unittest.mock import patch + +from django.contrib.auth.tokens import default_token_generator +from django.test import TestCase +from django.urls import reverse +from django.utils.encoding import force_bytes +from django.utils.http import urlsafe_base64_encode + +from dojo.models import Dojo_User, User, UserContactInfo +from dojo.user.authentication import reset_token_for_user + + +class TestUserUITimestamps(TestCase): + fixtures = ["dojo_testdata.json"] + + @patch("dojo.user.authentication.create_notification") + def test_view_user_contains_timestamps(self, mock_create_notification): + fixed = datetime(2025, 12, 12, 12, 0, 0, tzinfo=UTC) + admin = Dojo_User.objects.get(username="admin") + + # Create a target user and rotate their token at a fixed time. + target = User.objects.create(username="ui-ts-target", email="ui-ts-target@dojo.com") + with patch("dojo.user.authentication.timezone.now", return_value=fixed): + reset_token_for_user(acting_user=admin, target_user=target) + + # Ensure the UI can render and the timestamps match what we wrote. + self.client.force_login(admin) + resp = self.client.get(f"/user/{target.id}") + self.assertEqual(resp.status_code, 200) + viewed_user = resp.context["user"] + self.assertEqual(viewed_user.usercontactinfo.token_last_reset, fixed) + + # Now set password_last_reset to a fixed timestamp and assert it is exposed too. + uci, _ = UserContactInfo.objects.get_or_create(user=target) + uci.password_last_reset = fixed + uci.save(update_fields=["password_last_reset"]) + resp = self.client.get(f"/user/{target.id}") + viewed_user = resp.context["user"] + self.assertEqual(viewed_user.usercontactinfo.password_last_reset, fixed) + + def test_change_password_stamps_password_last_reset(self): + fixed = datetime(2025, 12, 12, 12, 0, 0, tzinfo=UTC) + user = User.objects.create(username="pwd-change-user", email="pwd-change-user@dojo.com", is_active=True) + user.set_password("OldPass123!@#") + user.save() + + self.client.force_login(user) + with patch("dojo.user.views.now", return_value=fixed): + resp = self.client.post( + reverse("change_password"), + data={ + "current_password": "OldPass123!@#", + "new_password": "NewPass123!@#", + "confirm_password": "NewPass123!@#", + }, + ) + # Successful change redirects to profile + self.assertEqual(resp.status_code, 302) + + uci = UserContactInfo.objects.get(user=user) + self.assertEqual(uci.password_last_reset, fixed) + + def test_password_reset_confirm_stamps_password_last_reset(self): + fixed = datetime(2025, 12, 12, 12, 0, 0, tzinfo=UTC) + user = User.objects.create(username="pwd-reset-user", email="pwd-reset-user@dojo.com", is_active=True) + user.set_password("OldPass123!@#") + user.save() + + uidb64 = urlsafe_base64_encode(force_bytes(user.pk)) + token = default_token_generator.make_token(user) + url = reverse("password_reset_confirm", kwargs={"uidb64": uidb64, "token": token}) + + with patch("dojo.user.views.now", return_value=fixed): + # Django's PasswordResetConfirmView typically requires a GET to the tokenized URL, + # which sets a session token and redirects to the "set-password" URL. + resp_get = self.client.get(url) + self.assertEqual(resp_get.status_code, 302) + set_password_url = resp_get["Location"] + + resp = self.client.post( + set_password_url, + data={ + "new_password1": "NewPass123!@#", + "new_password2": "NewPass123!@#", + }, + ) + + # PasswordResetConfirmView redirects to reset done on success + self.assertEqual(resp.status_code, 302) + uci = UserContactInfo.objects.get(user=user) + self.assertEqual(uci.password_last_reset, fixed) diff --git a/unittests/tools/test_api_bugcrowd_parser.py b/unittests/tools/test_api_bugcrowd_parser.py index 4b2c5436a97..ff15fa71820 100644 --- a/unittests/tools/test_api_bugcrowd_parser.py +++ b/unittests/tools/test_api_bugcrowd_parser.py @@ -152,5 +152,4 @@ def test_parse_file_with_broken_bug_url(self): parser = ApiBugcrowdParser() with self.assertLogs("dojo.tools.api_bugcrowd.parser", level="ERROR") as cm: parser.get_findings(testfile, Test()) - self.assertEqual(cm.output, ["ERROR:dojo.tools.api_bugcrowd.parser:" - "Error parsing bugcrowd bug_url : curl https://example.com/"]) + self.assertEqual(cm.output, ["ERROR:dojo.tools.api_bugcrowd.parser:Error parsing bugcrowd bug_url : curl https://example.com/"]) diff --git a/unittests/tools/test_pingcastle_parser.py b/unittests/tools/test_pingcastle_parser.py new file mode 100644 index 00000000000..64aece31447 --- /dev/null +++ b/unittests/tools/test_pingcastle_parser.py @@ -0,0 +1,49 @@ + +from dojo.tools.pingcastle.parser import PingCastleParser +from unittests.dojo_test_case import DojoTestCase, get_unit_tests_scans_path + + +class TestPingCastleParser(DojoTestCase): + + def test_no_findings(self): + with (get_unit_tests_scans_path("pingcastle") / "zero.xml").open(encoding="utf-8") as my_file_handle: + parser = PingCastleParser() + findings = parser.get_findings(my_file_handle, None) + self.assertEqual(0, len(findings)) + + def test_one_finding(self): + with (get_unit_tests_scans_path("pingcastle") / "one.xml").open(encoding="utf-8") as my_file_handle: + parser = PingCastleParser() + findings = parser.get_findings(my_file_handle, None) + self.assertEqual(1, len(findings)) + self.assertEqual(findings[0].title, "[PingCastle] A-MinPwdLen (Anomalies/WeakPassword)") + self.assertEqual(findings[0].severity, "Medium") + self.assertTrue(hasattr(findings[0], "unsaved_endpoints")) + + def test_many_findings(self): + with (get_unit_tests_scans_path("pingcastle") / "many.xml").open(encoding="utf-8") as my_file_handle: + parser = PingCastleParser() + findings = parser.get_findings(my_file_handle, None) + self.assertEqual(28, len(findings)) + admin_login = next((f for f in findings if f.vuln_id_from_tool == "P-AdminLogin"), None) + self.assertIsNotNone(admin_login) + self.assertEqual(admin_login.title, "[PingCastle] P-AdminLogin (PrivilegedAccounts/AdminControl)") + self.assertEqual(admin_login.severity, "Critical") + spooler = next((f for f in findings if f.vuln_id_from_tool == "A-DC-Spooler"), None) + self.assertIsNotNone(spooler) + self.assertEqual(spooler.title, "[PingCastle] A-DC-Spooler (Anomalies/PassTheCredential)") + self.assertEqual(spooler.severity, "High") + self.assertTrue(len(getattr(spooler, "unsaved_endpoints", [])) > 0) + for endpoint in spooler.unsaved_endpoints: + endpoint.clean() + ds_heuristics = next((f for f in findings if f.vuln_id_from_tool == "A-DsHeuristicsLDAPSecurity"), None) + self.assertIsNotNone(ds_heuristics) + self.assertEqual(ds_heuristics.title, "[PingCastle] A-DsHeuristicsLDAPSecurity (Anomalies/Reconnaissance)") + self.assertEqual(ds_heuristics.severity, "Medium") + self.assertTrue( + hasattr(ds_heuristics, "unsaved_vulnerability_ids") + and len(ds_heuristics.unsaved_vulnerability_ids) >= 1, + ) + coerce = next((f for f in findings if f.vuln_id_from_tool == "A-DC-Coerce"), None) + self.assertIsNotNone(coerce) + self.assertEqual(coerce.severity, "High") diff --git a/unittests/tools/test_prowler_parser.py b/unittests/tools/test_prowler_parser.py new file mode 100644 index 00000000000..0efc86dfbc6 --- /dev/null +++ b/unittests/tools/test_prowler_parser.py @@ -0,0 +1,932 @@ +from django.test import TestCase + +from dojo.models import Test +from dojo.tools.prowler.parser import ProwlerParser +from unittests.dojo_test_case import get_unit_tests_scans_path + + +class TestProwlerParser(TestCase): + + def test_prowler_parser_json_with_no_vuln_has_no_findings(self): + with (get_unit_tests_scans_path("prowler") / "prowler_zero_vul.json").open(encoding="utf-8") as testfile: + parser = ProwlerParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(0, len(findings)) + + def test_prowler_parser_csv_with_no_vuln_has_no_findings(self): + with (get_unit_tests_scans_path("prowler") / "prowler_zero_vul.csv").open(encoding="utf-8") as testfile: + parser = ProwlerParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(0, len(findings)) + + def test_prowler_parser_aws_csv_file_with_multiple_vulnerabilities(self): + with (get_unit_tests_scans_path("prowler") / "example_output_aws.csv").open(encoding="utf-8") as testfile: + parser = ProwlerParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(4, len(findings)) + with self.subTest(i=0): + description = ( + "**Cloud Type** : AWS\n\n" + "**Description** : Check if IAM Access Analyzer is enabled\n\n" + "**Service Name** : accessanalyzer\n\n" + "**Status Detail** : IAM Access Analyzer in account is not enabled.\n\n" + "**Finding Created Time** : 2025-02-14 14:27:03.913874\n\n" + "**Region** : \n\n" + "**Notes** : \n\n" + "**Related URL** : https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html\n\n" + "**Additional URLs** : https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-getting-started.html | https://aws.amazon.com/iam/features/analyze-access/" + ) + mitigation = ( + "**Remediation Recommendation** : Enable IAM Access Analyzer for all accounts, create analyzer and take action over it is recommendations (IAM Access Analyzer is available at no additional cost).\n\n" + "**Remediation Recommendation URL** : https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html\n\n" + "**Remediation Code Native IaC** : \n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : aws accessanalyzer create-analyzer --analyzer-name --type \n\n" + "**Other Remediation Info** : " + ) + references = ( + "CIS-1.4: 1.20\n" + "CIS-1.5: 1.20\n" + "KISA-ISMS-P-2023: 2.5.6, 2.6.4, 2.8.1, 2.8.2\n" + "CIS-2.0: 1.20\n" + "KISA-ISMS-P-2023-korean: 2.5.6, 2.6.4, 2.8.1, 2.8.2\n" + "AWS-Account-Security-Onboarding: Enabled security services, Create analyzers in each active regions, Verify that events are present in SecurityHub aggregated view\n" + "CIS-3.0: 1.20" + ) + i = 0 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Check if IAM Access Analyzer is enabled") + self.assertEqual(findings[i].severity, "Low") + self.assertEqual(findings[i].impact, "AWS IAM Access Analyzer helps you identify the resources in your organization and accounts, such as Amazon S3 buckets or IAM roles, that are shared with an external entity. This lets you identify unintended access to your resources and data, which is a security risk. IAM Access Analyzer uses a form of mathematical analysis called automated reasoning, which applies logic and mathematical inference to determine all possible access paths allowed by a resource policy.") + self.assertEqual(findings[i].references, references) + + with self.subTest(i=1): + description = ( + "**Cloud Type** : AWS\n\n" + "**Description** : Maintain current contact details.\n\n" + "**Service Name** : account\n\n" + "**Status Detail** : Login to the AWS Console. Choose your account name on the top right of the window -> My Account -> Contact Information.\n\n" + "**Finding Created Time** : 2025-02-14 14:27:03.913874\n\n" + "**Region** : \n\n" + "**Notes** : \n\n" + "**Additional URLs** : https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-getting-started.html | https://aws.amazon.com/iam/features/analyze-access/" + ) + mitigation = ( + "**Remediation Recommendation** : Using the Billing and Cost Management console complete contact details.\n\n" + "**Remediation Recommendation URL** : https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html\n\n" + "**Remediation Code Native IaC** : \n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : No command available.\n\n" + "**Other Remediation Info** : https://docs.prowler.com/checks/aws/iam-policies/iam_18-maintain-contact-details#aws-console" + ) + references = ( + "CIS-1.4: 1.1\n" + "CIS-1.5: 1.1\n" + "KISA-ISMS-P-2023: 2.1.3\n" + "CIS-2.0: 1.1\n" + "KISA-ISMS-P-2023-korean: 2.1.3\n" + "AWS-Well-Architected-Framework-Security-Pillar: SEC03-BP03, SEC10-BP01\n" + "AWS-Account-Security-Onboarding: Billing, emergency, security contacts\n" + "CIS-3.0: 1.1\n" + "ENS-RD2022: op.ext.7.aws.am.1" + ) + i = 1 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Maintain current contact details.") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization. An AWS account supports a number of contact details, and AWS will use these to contact the account owner if activity judged to be in breach of Acceptable Use Policy. If an AWS account is observed to be behaving in a prohibited or suspicious manner, AWS will attempt to contact the account owner by email and phone using the contact details listed. If this is unsuccessful and the account behavior needs urgent mitigation, proactive measures may be taken, including throttling of traffic between the account exhibiting suspicious behavior and the AWS API endpoints and the Internet. This will result in impaired service to and from the account in question.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=2): + description = ( + "**Cloud Type** : AWS\n\n" + "**Description** : Maintain different contact details to security, billing and operations.\n\n" + "**Service Name** : account\n\n" + "**Status Detail** : SECURITY, BILLING and OPERATIONS contacts not found or they are not different between each other and between ROOT contact.\n\n" + "**Finding Created Time** : 2025-02-14 14:27:03.913874\n\n" + "**Region** : \n\n" + "**Notes** : \n\n" + "**Related URL** : https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html\n\n" + "**Additional URLs** : https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-getting-started.html | https://aws.amazon.com/iam/features/analyze-access/" + ) + + mitigation = ( + "**Remediation Recommendation** : Using the Billing and Cost Management console complete contact details.\n\n" + "**Remediation Recommendation URL** : https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html\n\n" + "**Remediation Code Native IaC** : \n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : \n\n" + "**Other Remediation Info** : https://docs.prowler.com/checks/aws/iam-policies/iam_18-maintain-contact-details#aws-console" + ) + + references = ( + "KISA-ISMS-P-2023: 2.1.3\n" + "KISA-ISMS-P-2023-korean: 2.1.3" + ) + + i = 2 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Maintain different contact details to security, billing and operations.") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization. An AWS account supports a number of contact details, and AWS will use these to contact the account owner if activity judged to be in breach of Acceptable Use Policy. If an AWS account is observed to be behaving in a prohibited or suspicious manner, AWS will attempt to contact the account owner by email and phone using the contact details listed. If this is unsuccessful and the account behavior needs urgent mitigation, proactive measures may be taken, including throttling of traffic between the account exhibiting suspicious behavior and the AWS API endpoints and the Internet. This will result in impaired service to and from the account in question.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=3): + description = ( + "**Cloud Type** : AWS\n\n" + "**Description** : Ensure security contact information is registered.\n\n" + "**Service Name** : account\n\n" + "**Status Detail** : Login to the AWS Console. Choose your account name on the top right of the window -> My Account -> Alternate Contacts -> Security Section.\n\n" + "**Finding Created Time** : 2025-02-14 14:27:03.913874\n\n" + "**Region** : \n\n" + "**Notes** : \n\n" + "**Additional URLs** : https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-getting-started.html | https://aws.amazon.com/iam/features/analyze-access/" + ) + + mitigation = ( + "**Remediation Recommendation** : Go to the My Account section and complete alternate contacts.\n\n" + "**Remediation Recommendation URL** : https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html\n\n" + "**Remediation Code Native IaC** : \n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : No command available.\n\n" + "**Other Remediation Info** : https://docs.prowler.com/checks/aws/iam-policies/iam_19#aws-console" + ) + + references = ( + "CIS-1.4: 1.2\n" + "CIS-1.5: 1.2\n" + "AWS-Foundational-Security-Best-Practices: account, acm\n" + "KISA-ISMS-P-2023: 2.1.3, 2.2.1\n" + "CIS-2.0: 1.2\n" + "KISA-ISMS-P-2023-korean: 2.1.3, 2.2.1\n" + "AWS-Well-Architected-Framework-Security-Pillar: SEC03-BP03, SEC10-BP01\n" + "AWS-Account-Security-Onboarding: Billing, emergency, security contacts\n" + "CIS-3.0: 1.2\n" + "ENS-RD2022: op.ext.7.aws.am.1" + ) + + i = 3 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure security contact information is registered.") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "AWS provides customers with the option of specifying the contact information for accounts security team. It is recommended that this information be provided. Specifying security-specific contact information will help ensure that security advisories sent by AWS reach the team in your organization that is best equipped to respond to them.") + self.assertEqual(findings[i].references, references) + + def test_prowler_parser_azure_csv_file_with_multiple_vulnerabilities(self): + with (get_unit_tests_scans_path("prowler") / "example_output_azure.csv").open(encoding="utf-8") as testfile: + parser = ProwlerParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(4, len(findings)) + with self.subTest(i=0): + description = ( + "**Cloud Type** : AZURE\n\n" + "**Description** : Azure Kubernetes Service (AKS) can be configured to use Azure Active Directory (AD) for user authentication. In this configuration, you sign in to an AKS cluster using an Azure AD authentication token. You can also configure Kubernetes role-based access control (Kubernetes RBAC) to limit access to cluster resources based a user's identity or group membership.\n\n" + "**Service Name** : aks\n\n" + "**Status Detail** : RBAC is enabled for cluster '' in subscription ''.\n\n" + "**Finding Created Time** : 2025-02-14 14:27:30.710664\n\n" + "**Region** : \n\n" + "**Notes** : \n\n" + "**Related URL** : https://learn.microsoft.com/en-us/azure/aks/azure-ad-rbac?tabs=portal\n\n" + "**Additional URLs** : https://learn.microsoft.com/azure/aks/azure-ad-rbac | https://learn.microsoft.com/azure/aks/concepts-identity" + ) + + mitigation = ( + "**Remediation Recommendation** : \n\n" + "**Remediation Recommendation URL** : https://learn.microsoft.com/en-us/security/benchmark/azure/security-controls-v2-privileged-access#pa-7-follow-just-enough-administration-least-privilege-principle\n\n" + "**Remediation Code Native IaC** : \n\n" + "**Remediation Code Terraform** : https://docs.prowler.com/checks/azure/azure-kubernetes-policies/bc_azr_kubernetes_2#terraform\n\n" + "**Remediation Code CLI** : \n\n" + "**Other Remediation Info** : https://www.trendmicro.com/cloudoneconformity/knowledge-base/azure/AKS/enable-role-based-access-control-for-kubernetes-service.html#" + ) + + references = ( + "ENS-RD2022: op.acc.2.az.r1.eid.1" + ) + + i = 0 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure AKS RBAC is enabled") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Kubernetes RBAC and AKS help you secure your cluster access and provide only the minimum required permissions to developers and operators.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=1): + description = ( + "**Cloud Type** : AZURE\n\n" + "**Description** : Disable public IP addresses for cluster nodes, so that they only have private IP addresses. Private Nodes are nodes with no public IP addresses.\n\n" + "**Service Name** : aks\n\n" + "**Status Detail** : Cluster '' was created with private nodes in subscription ''\n\n" + "**Finding Created Time** : 2025-02-14 14:27:30.710664\n\n" + "**Region** : \n\n" + "**Notes** : \n\n" + "**Related URL** : https://learn.microsoft.com/en-us/azure/aks/private-clusters\n\n" + "**Additional URLs** : https://learn.microsoft.com/azure/aks/azure-ad-rbac | https://learn.microsoft.com/azure/aks/concepts-identity" + ) + mitigation = ( + "**Remediation Recommendation** : \n\n" + "**Remediation Recommendation URL** : https://learn.microsoft.com/en-us/azure/aks/access-private-cluster\n\n" + "**Remediation Code Native IaC** : \n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : \n\n" + "**Other Remediation Info** : " + ) + references = ( + "ENS-RD2022: mp.com.4.r2.az.aks.1\n" + "MITRE-ATTACK: T1190, T1530" + ) + + i = 1 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure clusters are created with Private Nodes") + self.assertEqual(findings[i].severity, "High") + self.assertEqual(findings[i].impact, "Disabling public IP addresses on cluster nodes restricts access to only internal networks, forcing attackers to obtain local network access before attempting to compromise the underlying Kubernetes hosts.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=2): + description = ( + "**Cloud Type** : AZURE\n\n" + "**Description** : Disable access to the Kubernetes API from outside the node network if it is not required.\n\n" + "**Service Name** : aks\n\n" + "**Status Detail** : Public access to nodes is enabled for cluster '' in subscription ''\n\n" + "**Finding Created Time** : 2025-02-14 14:27:30.710664\n\n" + "**Region** : \n\n" + "**Notes** : \n\n" + "**Related URL** : https://learn.microsoft.com/en-us/azure/aks/private-clusters?tabs=azure-portal\n\n" + "**Additional URLs** : https://learn.microsoft.com/azure/aks/azure-ad-rbac | https://learn.microsoft.com/azure/aks/concepts-identity" + ) + + mitigation = ( + "**Remediation Recommendation** : To use a private endpoint, create a new private endpoint in your virtual network then create a link between your virtual network and a new private DNS zone\n\n" + "**Remediation Recommendation URL** : https://learn.microsoft.com/en-us/azure/aks/access-private-cluster?tabs=azure-cli\n\n" + "**Remediation Code Native IaC** : \n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : az aks update -n -g --disable-public-fqdn\n\n" + "**Other Remediation Info** : " + ) + + references = ( + "ENS-RD2022: mp.com.4.az.aks.2\n" + "MITRE-ATTACK: T1190, T1530" + ) + + i = 2 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure clusters are created with Private Endpoint Enabled and Public Access Disabled") + self.assertEqual(findings[i].severity, "High") + self.assertEqual(findings[i].impact, "In a private cluster, the master node has two endpoints, a private and public endpoint. The private endpoint is the internal IP address of the master, behind an internal load balancer in the master's wirtual network. Nodes communicate with the master using the private endpoint. The public endpoint enables the Kubernetes API to be accessed from outside the master's virtual network. Although Kubernetes API requires an authorized token to perform sensitive actions, a vulnerability could potentially expose the Kubernetes publically with unrestricted access. Additionally, an attacker may be able to identify the current cluster and Kubernetes API version and determine whether it is vulnerable to an attack. Unless required, disabling public endpoint will help prevent such threats, and require the attacker to be on the master's virtual network to perform any attack on the Kubernetes API.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=3): + description = ( + "**Cloud Type** : AZURE\n\n" + "**Description** : When you run modern, microservices-based applications in Kubernetes, you often want to control which components can communicate with each other. The principle of least privilege should be applied to how traffic can flow between pods in an Azure Kubernetes Service (AKS) cluster. Let's say you likely want to block traffic directly to back-end applications. The Network Policy feature in Kubernetes lets you define rules for ingress and egress traffic between pods in a cluster.\n\n" + "**Service Name** : aks\n\n" + "**Status Detail** : Network policy is enabled for cluster '' in subscription ''.\n\n" + "**Finding Created Time** : 2025-02-14 14:27:30.710664\n\n" + "**Region** : \n\n" + "**Notes** : Network Policy requires the Network Policy add-on. This add-on is included automatically when a cluster with Network Policy is created, but for an existing cluster, needs to be added prior to enabling Network Policy. Enabling/Disabling Network Policy causes a rolling update of all cluster nodes, similar to performing a cluster upgrade. This operation is long-running and will block other operations on the cluster (including delete) until it has run to completion. If Network Policy is used, a cluster must have at least 2 nodes of type n1-standard-1 or higher. The recommended minimum size cluster to run Network Policy enforcement is 3 n1-standard-1 instances. Enabling Network Policy enforcement consumes additional resources in nodes. Specifically, it increases the memory footprint of the kube-system process by approximately 128MB, and requires approximately 300 millicores of CPU.\n\n" + "**Related URL** : https://learn.microsoft.com/en-us/security/benchmark/azure/security-controls-v2-network-security#ns-2-connect-private-networks-together\n\n" + "**Additional URLs** : https://learn.microsoft.com/azure/aks/azure-ad-rbac | https://learn.microsoft.com/azure/aks/concepts-identity" + ) + + mitigation = ( + "**Remediation Recommendation** : \n\n" + "**Remediation Recommendation URL** : https://learn.microsoft.com/en-us/azure/aks/use-network-policies\n\n" + "**Remediation Code Native IaC** : \n\n" + "**Remediation Code Terraform** : https://docs.prowler.com/checks/azure/azure-kubernetes-policies/bc_azr_kubernetes_4#terraform\n\n" + "**Remediation Code CLI** : \n\n" + "**Other Remediation Info** : " + ) + + references = ( + "ENS-RD2022: mp.com.4.r2.az.aks.1" + ) + + i = 3 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure Network Policy is Enabled and set as appropriate") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "All pods in an AKS cluster can send and receive traffic without limitations, by default. To improve security, you can define rules that control the flow of traffic. Back-end applications are often only exposed to required front-end services, for example. Or, database components are only accessible to the application tiers that connect to them. Network Policy is a Kubernetes specification that defines access policies for communication between Pods. Using Network Policies, you define an ordered set of rules to send and receive traffic and apply them to a collection of pods that match one or more label selectors. These network policy rules are defined as YAML manifests. Network policies can be included as part of a wider manifest that also creates a deployment or service.") + self.assertEqual(findings[i].references, references) + + def test_prowler_parser_gcp_csv_file_with_multiple_vulnerabilities(self): + with (get_unit_tests_scans_path("prowler") / "example_output_gcp.csv").open(encoding="utf-8") as testfile: + parser = ProwlerParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(2, len(findings)) + with self.subTest(i=0): + description = ( + "**Cloud Type** : GCP\n\n" + "**Description** : Scan images stored in Google Container Registry (GCR) for vulnerabilities using AR Container Analysis or a third-party provider. This helps identify and mitigate security risks associated with known vulnerabilities in container images.\n\n" + "**Service Name** : artifacts\n\n" + "**Status Detail** : AR Container Analysis is not enabled in project .\n\n" + "**Finding Created Time** : 2025-02-14 14:27:20.697446\n\n" + "**Region** : \n\n" + "**Notes** : By default, AR Container Analysis is disabled.\n\n" + "**Related URL** : https://cloud.google.com/artifact-analysis/docs\n\n" + "**Additional URLs** : https://cloud.google.com/api-keys/docs/best-practices | https://cloud.google.com/docs/authentication" + ) + + mitigation = ( + "**Remediation Recommendation** : Enable vulnerability scanning for images stored in Artifact Registry using AR Container Analysis or a third-party provider.\n\n" + "**Remediation Recommendation URL** : https://cloud.google.com/artifact-analysis/docs/container-scanning-overview\n\n" + "**Remediation Code Native IaC** : \n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : gcloud services enable containeranalysis.googleapis.com\n\n" + "**Other Remediation Info** : " + ) + + references = ( + "MITRE-ATTACK: T1525\n" + "ENS-RD2022: op.exp.4.r4.gcp.log.1, op.mon.3.gcp.scc.1" + ) + + i = 0 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure Image Vulnerability Analysis using AR Container Analysis or a third-party provider") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Without image vulnerability scanning, container images stored in Artifact Registry may contain known vulnerabilities, increasing the risk of exploitation by malicious actors.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=1): + description = ( + "**Cloud Type** : GCP\n\n" + "**Description** : GCP `Firewall Rules` are specific to a `VPC Network`. Each rule either `allows` or `denies` traffic when its conditions are met. Its conditions allow users to specify the type of traffic, such as ports and protocols, and the source or destination of the traffic, including IP addresses, subnets, and instances. Firewall rules are defined at the VPC network level and are specific to the network in which they are defined. The rules themselves cannot be shared among networks. Firewall rules only support IPv4 traffic. When specifying a source for an ingress rule or a destination for an egress rule by address, an `IPv4` address or `IPv4 block in CIDR` notation can be used. Generic `(0.0.0.0/0)` incoming traffic from the Internet to a VPC or VM instance using `RDP` on `Port 3389` can be avoided.\n\n" + "**Service Name** : networking\n\n" + "**Status Detail** : Firewall does not expose port 3389 (RDP) to the internet.\n\n" + "**Finding Created Time** : 2025-02-14 14:27:20.697446\n\n" + "**Region** : \n\n" + "**Notes** : \n\n" + "**Additional URLs** : https://cloud.google.com/api-keys/docs | https://cloud.google.com/docs/authentication" + ) + + mitigation = ( + "**Remediation Recommendation** : Ensure that Google Cloud Virtual Private Cloud (VPC) firewall rules do not allow unrestricted access (i.e. 0.0.0.0/0) on TCP port 3389 in order to restrict Remote Desktop Protocol (RDP) traffic to trusted IP addresses or IP ranges only and reduce the attack surface. TCP port 3389 is used for secure remote GUI login to Windows VM instances by connecting a RDP client application with an RDP server.\n\n" + "**Remediation Recommendation URL** : https://cloud.google.com/vpc/docs/using-firewalls\n\n" + "**Remediation Code Native IaC** : \n\n" + "**Remediation Code Terraform** : https://docs./checks/gcp/google-cloud-networking-policies/bc_gcp_networking_2#terraform\n\n" + "**Remediation Code CLI** : https://docs./checks/gcp/google-cloud-networking-policies/bc_gcp_networking_2#cli-command\n\n" + "**Other Remediation Info** : https://www.trendmicro.com/cloudoneconformity/knowledge-base/gcp/CloudVPC/unrestricted-rdp-access.html" + ) + + references = ( + "MITRE-ATTACK: T1190, T1199, T1048, T1498, T1046\n" + "CIS-2.0: 3.7\n" + "ENS-RD2022: mp.com.1.gcp.fw.1\n" + "CIS-3.0: 3.7" + ) + + i = 1 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure That RDP Access Is Restricted From the Internet") + self.assertEqual(findings[i].severity, "Critical") + self.assertEqual(findings[i].impact, "Allowing unrestricted Remote Desktop Protocol (RDP) access can increase opportunities for malicious activities such as hacking, Man-In-The-Middle attacks (MITM) and Pass-The-Hash (PTH) attacks.") + self.assertEqual(findings[i].references, references) + + def test_prowler_parser_kubernetes_csv_file_with_multiple_vulnerabilities(self): + with (get_unit_tests_scans_path("prowler") / "example_output_kubernetes.csv").open(encoding="utf-8") as testfile: + parser = ProwlerParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(4, len(findings)) + with self.subTest(i=0): + description = ( + "**Cloud Type** : KUBERNETES\n\n" + "**Description** : This check verifies that the AlwaysPullImages admission control plugin is enabled in the Kubernetes API server. This plugin ensures that every new pod always pulls the required images, enforcing image access control and preventing the use of possibly outdated or altered images.\n\n" + "**Service Name** : apiserver\n\n" + "**Status Detail** : AlwaysPullImages admission control plugin is not set in pod \n\n" + "**Finding Created Time** : 2025-02-14 14:27:38.533897\n\n" + "**Region** : namespace: kube-system\n\n" + "**Notes** : Enabling AlwaysPullImages can increase network and registry load and decrease container startup speed. It may not be suitable for all environments.\n\n" + "**Related URL** : https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#alwayspullimages\n\n" + "**Additional URLs** : https://kubernetes.io/docs/concepts/containers/images/ | https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/" + ) + + mitigation = ( + "**Remediation Recommendation** : Configure the API server to use the AlwaysPullImages admission control plugin to ensure image security and integrity.\n\n" + "**Remediation Recommendation URL** : https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers\n\n" + "**Remediation Code Native IaC** : https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-admission-control-plugin-alwayspullimages-is-set#kubernetes\n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : --enable-admission-plugins=...,AlwaysPullImages,...\n\n" + "**Other Remediation Info** : " + ) + + references = ( + "CIS-1.10: 1.2.11\n" + "CIS-1.8: 1.2.11" + ) + + i = 0 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure that the admission control plugin AlwaysPullImages is set") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Without AlwaysPullImages, once an image is pulled to a node, any pod can use it without any authorization check, potentially leading to security risks.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=1): + description = ( + "**Cloud Type** : KUBERNETES\n\n" + "**Description** : Disable anonymous requests to the API server. When enabled, requests that are not rejected by other configured authentication methods are treated as anonymous requests, which are then served by the API server. Disallowing anonymous requests strengthens security by ensuring all access is authenticated.\n\n" + "**Service Name** : apiserver\n\n" + "**Status Detail** : API Server does not have anonymous-auth enabled in pod \n\n" + "**Finding Created Time** : 2025-02-14 14:27:38.533897\n\n" + "**Region** : namespace: kube-system\n\n" + "**Notes** : While anonymous access can be useful for health checks and discovery, consider the security implications for your specific environment.\n\n" + "**Related URL** : https://kubernetes.io/docs/admin/authentication/#anonymous-requests\n\n" + "**Additional URLs** : https://kubernetes.io/docs/concepts/containers/images/ | https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/" + ) + + mitigation = ( + "**Remediation Recommendation** : Ensure the --anonymous-auth argument in the API server is set to false. This will reject all anonymous requests, enforcing authenticated access to the server.\n\n" + "**Remediation Recommendation URL** : https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/\n\n" + "**Remediation Code Native IaC** : https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-anonymous-auth-argument-is-set-to-false-1#kubernetes\n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : --anonymous-auth=false\n\n" + "**Other Remediation Info** : " + ) + + references = ( + "CIS-1.10: 1.2.1\n" + "CIS-1.8: 1.2.1" + ) + + i = 1 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure that the --anonymous-auth argument is set to false") + self.assertEqual(findings[i].severity, "High") + self.assertEqual(findings[i].impact, "Enabling anonymous access to the API server can expose the cluster to unauthorized access and potential security vulnerabilities.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=2): + description = ( + "**Cloud Type** : KUBERNETES\n\n" + "**Description** : This check ensures that the Kubernetes API server is configured with an appropriate audit log retention period. Setting --audit-log-maxage to 30 or as per business requirements helps in maintaining logs for sufficient time to investigate past events.\n\n" + "**Service Name** : apiserver\n\n" + "**Status Detail** : Audit log max age is not set to 30 or as appropriate in pod \n\n" + "**Finding Created Time** : 2025-02-14 14:27:38.533897\n\n" + "**Region** : namespace: kube-system\n\n" + "**Notes** : Ensure the audit log retention period is set appropriately to balance between storage constraints and the need for historical data.\n\n" + "**Related URL** : https://kubernetes.io/docs/concepts/cluster-administration/audit/\n\n" + "**Additional URLs** : https://kubernetes.io/docs/concepts/containers/images/ | https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/" + ) + + mitigation = ( + "**Remediation Recommendation** : Configure the API server audit log retention period to retain logs for at least 30 days or as per your organization's requirements.\n\n" + "**Remediation Recommendation URL** : https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/\n\n" + "**Remediation Code Native IaC** : https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-audit-log-maxage-argument-is-set-to-30-or-as-appropriate#kubernetes\n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : --audit-log-maxage=30\n\n" + "**Other Remediation Info** : " + ) + + references = ( + "CIS-1.10: 1.2.17\n" + "CIS-1.8: 1.2.18" + ) + + i = 2 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure that the --audit-log-maxage argument is set to 30 or as appropriate") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Without an adequate log retention period, there may be insufficient audit history to investigate and analyze past events or security incidents.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=3): + description = ( + "**Cloud Type** : KUBERNETES\n\n" + "**Description** : This check ensures that the Kubernetes API server is configured with an appropriate number of audit log backups. Setting --audit-log-maxbackup to 10 or as per business requirements helps maintain a sufficient log backup for investigations or analysis.\n\n" + "**Service Name** : apiserver\n\n" + "**Status Detail** : Audit log max backup is not set to 10 or as appropriate in pod \n\n" + "**Finding Created Time** : 2025-02-14 14:27:38.533897\n\n" + "**Region** : namespace: kube-system\n\n" + "**Notes** : Ensure the audit log backup retention period is set appropriately to balance between storage constraints and the need for historical data.\n\n" + "**Related URL** : https://kubernetes.io/docs/concepts/cluster-administration/audit/\n\n" + "**Additional URLs** : https://kubernetes.io/docs/concepts/containers/images/ | https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/" + ) + + mitigation = ( + "**Remediation Recommendation** : Configure the API server audit log backup retention to 10 or as per your organization's requirements.\n\n" + "**Remediation Recommendation URL** : https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/\n\n" + "**Remediation Code Native IaC** : https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate#kubernetes\n\n" + "**Remediation Code Terraform** : \n\n" + "**Remediation Code CLI** : --audit-log-maxbackup=10\n\n" + "**Other Remediation Info** : " + ) + + references = ( + "CIS-1.10: 1.2.18\n" + "CIS-1.8: 1.2.19" + ) + + i = 3 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Without an adequate number of audit log backups, there may be insufficient log history to investigate past events or security incidents.") + self.assertEqual(findings[i].references, references) + + def test_prowler_parser_aws_json_file_with_multiple_vulnerabilities(self): + with (get_unit_tests_scans_path("prowler") / "example_output_aws.ocsf.json").open(encoding="utf-8") as testfile: + parser = ProwlerParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(3, len(findings)) + with self.subTest(i=0): + description = ( + "**Cloud Type** : AWS\n\n" + "**Finding Description** : Check if IAM Access Analyzer is enabled\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : IAM Access Analyzer in account is not enabled.\n\n" + "**Finding Created Time** : 2025-02-14T14:27:03.913874\n\n" + "**AWS Region** : " + ) + + mitigation = ( + "**Remediation Description** : Enable IAM Access Analyzer for all accounts, create analyzer and take action over it is recommendations (IAM Access Analyzer is available at no additional cost).\n\n" + "**Remediation References** : aws accessanalyzer create-analyzer --analyzer-name --type , https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html" + ) + + references = ( + "**Related URL** : https://docs.aws.amazon.com/IAM/latest/UserGuide/what-is-access-analyzer.html\n\n" + "**CIS-1.4** : 1.20\n\n" + "**CIS-1.5** : 1.20\n\n" + "**KISA-ISMS-P-2023** : 2.5.6, 2.6.4, 2.8.1, 2.8.2\n\n" + "**CIS-2.0** : 1.20\n\n" + "**KISA-ISMS-P-2023-korean** : 2.5.6, 2.6.4, 2.8.1, 2.8.2\n\n" + "**AWS-Account-Security-Onboarding** : Enabled security services, Create analyzers in each active regions, Verify that events are present in SecurityHub aggregated view\n\n" + "**CIS-3.0** : 1.20" + ) + + i = 0 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "IAM Access Analyzer in account is not enabled.") + self.assertEqual(findings[i].severity, "Low") + self.assertEqual(findings[i].impact, "AWS IAM Access Analyzer helps you identify the resources in your organization and accounts, such as Amazon S3 buckets or IAM roles, that are shared with an external entity. This lets you identify unintended access to your resources and data, which is a security risk. IAM Access Analyzer uses a form of mathematical analysis called automated reasoning, which applies logic and mathematical inference to determine all possible access paths allowed by a resource policy.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=1): + description = ( + "**Cloud Type** : AWS\n\n" + "**Finding Description** : Maintain current contact details.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : Login to the AWS Console. Choose your account name on the top right of the window -> My Account -> Contact Information.\n\n" + "**Finding Created Time** : 2025-02-14T14:27:03.913874\n\n" + "**AWS Region** : " + ) + + mitigation = ( + "**Remediation Description** : Using the Billing and Cost Management console complete contact details.\n\n" + "**Remediation References** : No command available., https://docs.prowler.com/checks/aws/iam-policies/iam_18-maintain-contact-details#aws-console, https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html" + ) + + references = ( + "**Related URL** : \n\n" + "**CIS-1.4** : 1.1\n\n" + "**CIS-1.5** : 1.1\n\n" + "**KISA-ISMS-P-2023** : 2.1.3\n\n" + "**CIS-2.0** : 1.1\n\n" + "**KISA-ISMS-P-2023-korean** : 2.1.3\n\n" + "**AWS-Well-Architected-Framework-Security-Pillar** : SEC03-BP03, SEC10-BP01\n\n" + "**AWS-Account-Security-Onboarding** : Billing, emergency, security contacts\n\n" + "**CIS-3.0** : 1.1\n\n" + "**ENS-RD2022** : op.ext.7.aws.am.1" + ) + + i = 1 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Login to the AWS Console. Choose your account name on the top right of the window -> My Account -> Contact Information.") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization. An AWS account supports a number of contact details, and AWS will use these to contact the account owner if activity judged to be in breach of Acceptable Use Policy. If an AWS account is observed to be behaving in a prohibited or suspicious manner, AWS will attempt to contact the account owner by email and phone using the contact details listed. If this is unsuccessful and the account behavior needs urgent mitigation, proactive measures may be taken, including throttling of traffic between the account exhibiting suspicious behavior and the AWS API endpoints and the Internet. This will result in impaired service to and from the account in question.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=2): + description = ( + "**Cloud Type** : AWS\n\n" + "**Finding Description** : Maintain different contact details to security, billing and operations.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : SECURITY, BILLING and OPERATIONS contacts not found or they are not different between each other and between ROOT contact.\n\n" + "**Finding Created Time** : 2025-02-14T14:27:03.913874\n\n" + "**AWS Region** : " + ) + + mitigation = ( + "**Remediation Description** : Using the Billing and Cost Management console complete contact details.\n\n" + "**Remediation References** : https://docs.prowler.com/checks/aws/iam-policies/iam_18-maintain-contact-details#aws-console, https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html" + ) + + references = ( + "**Related URL** : https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-update-contact.html\n\n" + "**KISA-ISMS-P-2023** : 2.1.3\n\n" + "**KISA-ISMS-P-2023-korean** : 2.1.3" + ) + + i = 2 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "SECURITY, BILLING and OPERATIONS contacts not found or they are not different between each other and between ROOT contact.") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Ensure contact email and telephone details for AWS accounts are current and map to more than one individual in your organization. An AWS account supports a number of contact details, and AWS will use these to contact the account owner if activity judged to be in breach of Acceptable Use Policy. If an AWS account is observed to be behaving in a prohibited or suspicious manner, AWS will attempt to contact the account owner by email and phone using the contact details listed. If this is unsuccessful and the account behavior needs urgent mitigation, proactive measures may be taken, including throttling of traffic between the account exhibiting suspicious behavior and the AWS API endpoints and the Internet. This will result in impaired service to and from the account in question.") + self.assertEqual(findings[i].references, references) + + def test_prowler_parser_azure_json_file_with_multiple_vulnerabilities(self): + with (get_unit_tests_scans_path("prowler") / "example_output_azure.ocsf.json").open(encoding="utf-8") as testfile: + parser = ProwlerParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(3, len(findings)) + with self.subTest(i=0): + description = ( + "**Cloud Type** : AZURE\n\n" + "**Finding Description** : Application Insights within Azure act as an Application Performance Monitoring solution providing valuable data into how well an application performs and additional information when performing incident response. The types of log data collected include application metrics, telemetry data, and application trace logging data providing organizations with detailed information about application activity and application transactions. Both data sets help organizations adopt a proactive and retroactive means to handle security and performance related metrics within their modern applications.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : There are no AppInsight configured in subscription .\n\n" + "**Finding Created Time** : 2025-02-14T14:27:30.710664\n\n" + "**AZURE Region** : global" + ) + + mitigation = ( + "**Remediation Description** : 1. Navigate to Application Insights 2. Under the Basics tab within the PROJECT DETAILS section, select the Subscription 3. Select the Resource group 4. Within the INSTANCE DETAILS, enter a Name 5. Select a Region 6. Next to Resource Mode, select Workspace-based 7. Within the WORKSPACE DETAILS, select the Subscription for the log analytics workspace 8. Select the appropriate Log Analytics Workspace 9. Click Next:Tags > 10. Enter the appropriate Tags as Name, Value pairs. 11. Click Next:Review+Create 12. Click Create.\n\n" + "**Remediation References** : az monitor app-insights component create --app --resource-group --location --kind 'web' --retention-time --workspace -- subscription , https://www.tenable.com/audits/items/CIS_Microsoft_Azure_Foundations_v2.0.0_L2.audit:8a7a608d180042689ad9d3f16aa359f1" + ) + + references = ( + "**Related URL** : https://learn.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview\n\n" + "**CIS-2.1** : 5.3.1\n\n" + "**ENS-RD2022** : mp.s.4.r1.az.nt.2\n\n" + "**CIS-3.0** : 6.3.1\n\n" + "**CIS-2.0** : 5.3.1" + ) + + i = 0 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "There are no AppInsight configured in subscription .") + self.assertEqual(findings[i].severity, "Low") + self.assertEqual(findings[i].impact, "Configuring Application Insights provides additional data not found elsewhere within Azure as part of a much larger logging and monitoring program within an organization's Information Security practice. The types and contents of these logs will act as both a potential cost saving measure (application performance) and a means to potentially confirm the source of a potential incident (trace logging). Metrics and Telemetry data provide organizations with a proactive approach to cost savings by monitoring an application's performance, while the trace logging data provides necessary details in a reactive incident response scenario by helping organizations identify the potential source of an incident within their application.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=1): + description = ( + "**Cloud Type** : AZURE\n\n" + "**Finding Description** : Microsoft Defender for Cloud emails the subscription owners whenever a high-severity alert is triggered for their subscription. You should provide a security contact email address as an additional email address.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : There is not another correct email configured for subscription .\n\n" + "**Finding Created Time** : 2025-02-14T14:27:30.710664\n\n" + "**AZURE Region** : global" + ) + + mitigation = ( + "**Remediation Description** : 1. From Azure Home select the Portal Menu 2. Select Microsoft Defender for Cloud 3. Click on Environment Settings 4. Click on the appropriate Management Group, Subscription, or Workspace 5. Click on Email notifications 6. Enter a valid security contact email address (or multiple addresses separated by commas) in the Additional email addresses field 7. Click Save\n\n" + "**Remediation References** : https://docs.prowler.com/checks/azure/azure-general-policies/ensure-that-security-contact-emails-is-set#terraform, https://www.trendmicro.com/cloudoneconformity/knowledge-base/azure/SecurityCenter/security-contact-email.html, https://learn.microsoft.com/en-us/rest/api/defenderforcloud/security-contacts/list?view=rest-defenderforcloud-2020-01-01-preview&tabs=HTTP" + ) + + references = ( + "**Related URL** : https://docs.microsoft.com/en-us/azure/security-center/security-center-provide-security-contact-details\n\n" + "**CIS-2.1** : 2.1.18\n\n" + "**ENS-RD2022** : op.mon.3.r3.az.de.1\n\n" + "**CIS-3.0** : 3.1.13\n\n" + "**CIS-2.0** : 2.1.19" + ) + + i = 1 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "There is not another correct email configured for subscription .") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Microsoft Defender for Cloud emails the Subscription Owner to notify them about security alerts. Adding your Security Contact's email address to the 'Additional email addresses' field ensures that your organization's Security Team is included in these alerts. This ensures that the proper people are aware of any potential compromise in order to mitigate the risk in a timely fashion.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=2): + description = ( + "**Cloud Type** : AZURE\n\n" + "**Finding Description** : Ensure That Microsoft Defender for App Services Is Set To 'On' \n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : Defender plan Defender for App Services from subscription is set to OFF (pricing tier not standard).\n\n" + "**Finding Created Time** : 2025-02-14T14:27:30.710664\n\n" + "**AZURE Region** : global" + ) + + mitigation = ( + "**Remediation Description** : By , Microsoft Defender for Cloud is not enabled for your App Service instances. Enabling the Defender security service for App Service instances allows for advanced security defense using threat detection capabilities provided by Microsoft Security Response Center.\n\n" + "**Remediation References** : https://docs.prowler.com/checks/azure/azure-general-policies/ensure-that-azure-defender-is-set-to-on-for-app-service#terraform, https://www.trendmicro.com/cloudoneconformity/knowledge-base/azure/SecurityCenter/defender-app-service.html, https://www.trendmicro.com/cloudoneconformity/knowledge-base/azure/SecurityCenter/defender-app-service.html" + ) + + references = ( + "**Related URL** : \n\n" + "**CIS-2.1** : 2.1.2\n\n" + "**ENS-RD2022** : mp.s.4.r1.az.nt.3\n\n" + "**MITRE-ATTACK** : T1190, T1059, T1204, T1552, T1486, T1499, T1496, T1087\n\n" + "**CIS-3.0** : 3.1.6.1" + ) + + i = 2 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Defender plan Defender for App Services from subscription is set to OFF (pricing tier not standard).") + self.assertEqual(findings[i].severity, "High") + self.assertEqual(findings[i].impact, "Turning on Microsoft Defender for App Service enables threat detection for App Service, providing threat intelligence, anomaly detection, and behavior analytics in the Microsoft Defender for Cloud.") + self.assertEqual(findings[i].references, references) + + def test_prowler_parser_gcp_json_file_with_multiple_vulnerabilities(self): + with (get_unit_tests_scans_path("prowler") / "example_output_gcp.ocsf.json").open(encoding="utf-8") as testfile: + parser = ProwlerParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(3, len(findings)) + with self.subTest(i=0): + description = ( + "**Cloud Type** : GCP\n\n" + "**Finding Description** : API Keys should only be used for services in cases where other authentication methods are unavailable. Unused keys with their permissions in tact may still exist within a project. Keys are insecure because they can be viewed publicly, such as from within a browser, or they can be accessed on a device where the key resides. It is recommended to use standard authentication flow instead.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : Project does not have active API Keys.\n\n" + "**Finding Created Time** : 2025-02-14T14:27:20.697446\n\n" + "**GCP Region** : global" + ) + + mitigation = ( + "**Remediation Description** : To avoid the security risk in using API keys, it is recommended to use standard authentication flow instead.\n\n" + "**Remediation References** : gcloud alpha services api-keys delete, https://cloud.google.com/docs/authentication/api-keys" + ) + + references = ( + "**Related URL** : \n\n" + "**MITRE-ATTACK** : T1098\n\n" + "**CIS-2.0** : 1.12\n\n" + "**ENS-RD2022** : op.acc.2.gcp.rbak.1\n\n" + "**CIS-3.0** : 1.12" + ) + + i = 0 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Project does not have active API Keys.") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Security risks involved in using API-Keys appear below: API keys are simple encrypted strings, API keys do not identify the user or the application making the API request, API keys are typically accessible to clients, making it easy to discover and steal an API key.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=1): + description = ( + "**Cloud Type** : GCP\n\n" + "**Finding Description** : Scan images stored in Google Container Registry (GCR) for vulnerabilities using AR Container Analysis or a third-party provider. This helps identify and mitigate security risks associated with known vulnerabilities in container images.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : AR Container Analysis is not enabled in project .\n\n" + "**Finding Created Time** : 2025-02-14T14:27:20.697446\n\n" + "**GCP Region** : global" + ) + + mitigation = ( + "**Remediation Description** : Enable vulnerability scanning for images stored in Artifact Registry using AR Container Analysis or a third-party provider.\n\n" + "**Remediation References** : gcloud services enable containeranalysis.googleapis.com, https://cloud.google.com/artifact-analysis/docs/container-scanning-overview" + ) + + references = ( + "**Related URL** : https://cloud.google.com/artifact-analysis/docs\n\n" + "**MITRE-ATTACK** : T1525\n\n" + "**ENS-RD2022** : op.exp.4.r4.gcp.log.1, op.mon.3.gcp.scc.1" + ) + + i = 1 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "AR Container Analysis is not enabled in project .") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Without image vulnerability scanning, container images stored in Artifact Registry may contain known vulnerabilities, increasing the risk of exploitation by malicious actors.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=2): + description = ( + "**Cloud Type** : GCP\n\n" + "**Finding Description** : GCP `Firewall Rules` are specific to a `VPC Network`. Each rule either `allows` or `denies` traffic when its conditions are met. Its conditions allow users to specify the type of traffic, such as ports and protocols, and the source or destination of the traffic, including IP addresses, subnets, and instances. Firewall rules are defined at the VPC network level and are specific to the network in which they are defined. The rules themselves cannot be shared among networks. Firewall rules only support IPv4 traffic. When specifying a source for an ingress rule or a destination for an egress rule by address, an `IPv4` address or `IPv4 block in CIDR` notation can be used. Generic `(0.0.0.0/0)` incoming traffic from the Internet to a VPC or VM instance using `RDP` on `Port 3389` can be avoided.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : Firewall does exposes port 3389 (RDP) to the internet.\n\n" + "**Finding Created Time** : 2025-02-14T14:27:20.697446\n\n" + "**GCP Region** : global" + ) + + mitigation = ( + "**Remediation Description** : Ensure that Google Cloud Virtual Private Cloud (VPC) firewall rules do not allow unrestricted access (i.e. 0.0.0.0/0) on TCP port 3389 in order to restrict Remote Desktop Protocol (RDP) traffic to trusted IP addresses or IP ranges only and reduce the attack surface. TCP port 3389 is used for secure remote GUI login to Windows VM instances by connecting a RDP client application with an RDP server.\n\n" + "**Remediation References** : https://docs.prowler.com/checks/gcp/google-cloud-networking-policies/bc_gcp_networking_2#terraform, https://docs.prowler.com/checks/gcp/google-cloud-networking-policies/bc_gcp_networking_2#cli-command, https://www.trendmicro.com/cloudoneconformity/knowledge-base/gcp/CloudVPC/unrestricted-rdp-access.html, https://cloud.google.com/vpc/docs/using-firewalls" + ) + + references = ( + "**Related URL** : \n\n" + "**MITRE-ATTACK** : T1190, T1199, T1048, T1498, T1046\n\n" + "**CIS-2.0** : 3.7\n\n" + "**ENS-RD2022** : mp.com.1.gcp.fw.1\n\n" + "**CIS-3.0** : 3.7" + ) + + i = 2 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Firewall does exposes port 3389 (RDP) to the internet.") + self.assertEqual(findings[i].severity, "Critical") + self.assertEqual(findings[i].impact, "Allowing unrestricted Remote Desktop Protocol (RDP) access can increase opportunities for malicious activities such as hacking, Man-In-The-Middle attacks (MITM) and Pass-The-Hash (PTH) attacks.") + self.assertEqual(findings[i].references, references) + + def test_prowler_parser_kubernetes_json_file_with_multiple_vulnerabilities(self): + with (get_unit_tests_scans_path("prowler") / "example_output_kubernetes.ocsf.json").open(encoding="utf-8") as testfile: + parser = ProwlerParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(3, len(findings)) + with self.subTest(i=0): + description = ( + "**Cloud Type** : KUBERNETES\n\n" + "**Finding Description** : This check verifies that the AlwaysPullImages admission control plugin is enabled in the Kubernetes API server. This plugin ensures that every new pod always pulls the required images, enforcing image access control and preventing the use of possibly outdated or altered images.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : AlwaysPullImages admission control plugin is not set in pod .\n\n" + "**Finding Created Time** : 2025-02-14T14:27:38.533897\n\n" + "**Pod Name** : \n\n" + "**Namespace** : " + ) + + mitigation = ( + "**Remediation Description** : Configure the API server to use the AlwaysPullImages admission control plugin to ensure image security and integrity.\n\n" + "**Remediation References** : https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-admission-control-plugin-alwayspullimages-is-set#kubernetes, --enable-admission-plugins=...,AlwaysPullImages,..., https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers" + ) + + references = ( + "**Related URL** : https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#alwayspullimages\n\n" + "**CIS-1.10** : 1.2.11\n\n" + "**CIS-1.8** : 1.2.11" + ) + + i = 0 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "AlwaysPullImages admission control plugin is not set in pod .") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Without AlwaysPullImages, once an image is pulled to a node, any pod can use it without any authorization check, potentially leading to security risks.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=1): + description = ( + "**Cloud Type** : KUBERNETES\n\n" + "**Finding Description** : This check ensures that the Kubernetes API server is configured with an appropriate audit log retention period. Setting --audit-log-maxage to 30 or as per business requirements helps in maintaining logs for sufficient time to investigate past events.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : Audit log max age is not set to 30 or as appropriate in pod .\n\n" + "**Finding Created Time** : 2025-02-14T14:27:38.533897\n\n" + "**Pod Name** : \n\n" + "**Namespace** : " + ) + + mitigation = ( + "**Remediation Description** : Configure the API server audit log retention period to retain logs for at least 30 days or as per your organization's requirements.\n\n" + "**Remediation References** : https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-audit-log-maxage-argument-is-set-to-30-or-as-appropriate#kubernetes, --audit-log-maxage=30, https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/" + ) + + references = ( + "**Related URL** : https://kubernetes.io/docs/concepts/cluster-administration/audit/\n\n" + "**CIS-1.10** : 1.2.17\n\n" + "**CIS-1.8** : 1.2.18" + ) + + i = 1 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Audit log max age is not set to 30 or as appropriate in pod .") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Without an adequate log retention period, there may be insufficient audit history to investigate and analyze past events or security incidents.") + self.assertEqual(findings[i].references, references) + with self.subTest(i=2): + description = ( + "**Cloud Type** : KUBERNETES\n\n" + "**Finding Description** : This check ensures that the Kubernetes API server is configured with an appropriate number of audit log backups. Setting --audit-log-maxbackup to 10 or as per business requirements helps maintain a sufficient log backup for investigations or analysis.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : Audit log max backup is not set to 10 or as appropriate in pod .\n\n" + "**Finding Created Time** : 2025-02-14T14:27:38.533897\n\n" + "**Pod Name** : \n\n" + "**Namespace** : \n" + "**Cloud Type** : KUBERNETES\n\n" + "**Finding Description** : This check ensures that the Kubernetes API server is configured with an appropriate number of audit log backups. Setting --audit-log-maxbackup to 10 or as per business requirements helps maintain a sufficient log backup for investigations or analysis.\n\n" + "**Product Name** : Prowler\n\n" + "**Status Detail** : Audit log max backup is not set to 10 or as appropriate in pod .\n\n" + "**Finding Created Time** : 2025-02-14T14:27:38.533897\n\n" + "**Pod Name** : \n\n" + "**Namespace** : " + ) + + mitigation = ( + "**Remediation Description** : Configure the API server audit log backup retention to 10 or as per your organization's requirements.\n\n" + "**Remediation References** : https://docs.prowler.com/checks/kubernetes/kubernetes-policy-index/ensure-that-the-audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate#kubernetes, --audit-log-maxbackup=10, https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/" + ) + + references = ( + "**Related URL** : https://kubernetes.io/docs/concepts/cluster-administration/audit/\n\n" + "**CIS-1.10** : 1.2.18\n\n" + "**CIS-1.8** : 1.2.19" + ) + + i = 2 + self.assertEqual(findings[i].description, description) + self.assertEqual(findings[i].mitigation, mitigation) + self.assertEqual(findings[i].title, "Audit log max backup is not set to 10 or as appropriate in pod .") + self.assertEqual(findings[i].severity, "Medium") + self.assertEqual(findings[i].impact, "Without an adequate number of audit log backups, there may be insufficient log history to investigate past events or security incidents.") + self.assertEqual(findings[i].references, references) diff --git a/unittests/tools/test_snyk_issue_api_parser_with_json.py b/unittests/tools/test_snyk_issue_api_parser_with_json.py index ae61dc0cf12..16473bb0e55 100644 --- a/unittests/tools/test_snyk_issue_api_parser_with_json.py +++ b/unittests/tools/test_snyk_issue_api_parser_with_json.py @@ -7,9 +7,9 @@ class TestSnykIssueApiParserWithJson(DojoTestCase): def parse_json(self, filename): - testfile = (get_unit_tests_scans_path("snyk_issue_api") / filename).open(encoding="utf-8") - parser = SnykIssueApiParser() - return parser.get_findings(testfile, Test()) + with (get_unit_tests_scans_path("snyk_issue_api") / filename).open(encoding="utf-8") as testfile: + parser = SnykIssueApiParser() + return parser.get_findings(testfile, Test()) def test_parse_sca_single_finding(self): findings = self.parse_json("snyk_sca_scan_api_single_vuln.json")