diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65ada228f..bcf1dc561 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,6 +63,7 @@ jobs: python -m coverage xml - name: Run analysis on SonarQube + if: ${{ github.event.pull_request.head.repo.fork == false }} uses: SonarSource/sonarqube-scan-action@v2 env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} @@ -125,4 +126,4 @@ jobs: - name: Semantic Release run: npx semantic-release@23.0.0 env: - GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} \ No newline at end of file + GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} diff --git a/.github/workflows/intellij-build.yml b/.github/workflows/intellij-build.yml index 36c2d0e2c..b6e961cc5 100644 --- a/.github/workflows/intellij-build.yml +++ b/.github/workflows/intellij-build.yml @@ -67,14 +67,8 @@ jobs: - name: Setup Java uses: actions/setup-java@v4 with: - distribution: zulu - java-version: 17 - - # Setup Gradle - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - with: - gradle-home-cache-cleanup: true + distribution: temurin + java-version: 21 # Set environment variables - name: Export Properties @@ -121,14 +115,8 @@ jobs: - name: Setup Java uses: actions/setup-java@v4 with: - distribution: zulu - java-version: 17 - - # Setup Gradle - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - with: - gradle-home-cache-cleanup: true + distribution: temurin + java-version: 21 # Run tests - name: Run Tests @@ -164,21 +152,8 @@ jobs: - name: Setup Java uses: actions/setup-java@v4 with: - distribution: zulu - java-version: 17 - - # Setup Gradle - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - with: - gradle-home-cache-cleanup: true - - # Cache Plugin Verifier IDEs - - name: Setup Plugin Verifier IDEs Cache - uses: actions/cache@v4 - with: - path: ${{ needs.build.outputs.pluginVerifierHomeDir }}/ides - key: plugin-verifier-${{ hashFiles('build/listProductsReleases.txt') }} + distribution: temurin + java-version: 21 # Run Verify Plugin task and IntelliJ Plugin Verifier tool - name: Run Plugin Verification tasks diff --git a/.github/workflows/intellij-release.yml b/.github/workflows/intellij-release.yml index 3593c55c7..a25483dfd 100644 --- a/.github/workflows/intellij-release.yml +++ b/.github/workflows/intellij-release.yml @@ -37,14 +37,8 @@ jobs: - name: Setup Java uses: actions/setup-java@v4 with: - distribution: zulu - java-version: 17 - - # Setup Gradle - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - with: - gradle-home-cache-cleanup: true + distribution: temurin + java-version: 21 - name: Set env run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/intellij-v}" >> $GITHUB_ENV diff --git a/.github/workflows/intellij-updater.yml b/.github/workflows/intellij-updater.yml index a2cb7d7a6..f2f7e2ed0 100644 --- a/.github/workflows/intellij-updater.yml +++ b/.github/workflows/intellij-updater.yml @@ -25,8 +25,8 @@ jobs: - name: Set up JDK 17 uses: actions/setup-java@2dfa2011c5b2a0f1489bf9e433881c92c1631f88 # v4.3.0 with: - distribution: 'temurin' - java-version: 17 + distribution: temurin + java-version: 21 - name: Set up NodeJS Latest uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 #v4.0.4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a4205795..6e27da4dc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,6 +74,7 @@ jobs: packages-dir: tools/dist/ - name: Build and tag Docker image + timeout-minutes: 2 run: | docker build --build-arg VERSION=${{ env.RELEASE_VERSION }} -t bancolombia/devsecops-engine-tools:${{ env.RELEASE_VERSION }} -f docker/Dockerfile . docker tag bancolombia/devsecops-engine-tools:${{ env.RELEASE_VERSION }} bancolombia/devsecops-engine-tools:${{ env.RELEASE_VERSION }} diff --git a/.gitignore b/.gitignore index 184064f3c..12c7940f3 100644 --- a/.gitignore +++ b/.gitignore @@ -39,15 +39,12 @@ prueba.py main.py common_devsecops_lib/sheckov_scan.json _core_test* -results.json -bearer*.yml -bearer*.json -bin results*.json RULES_DOCKERcheckov_config.yaml RULES_K8Scheckov_config.yaml RULES_CLOUDFORMATIONcheckov_config.yaml RULES_OPENAPIcheckov_config.yaml +RULES_TERRAFORMcheckov_config.yaml log/ twistcli trivy* @@ -64,6 +61,12 @@ kics_assets/ assets_compressed.zip kics_*.zip kubescape-macos-latest +*.tar.gz +*.zip +*_SBOM.json +syft +gitleaks_aux_report_*.json +gitleaks_report.json # Extensions out \ No newline at end of file diff --git a/README.md b/README.md index f01389e12..f79a4a4f3 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=bancolombia_devsecops-engine-tools&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=bancolombia_devsecops-engine-tools) [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=bancolombia_devsecops-engine-tools&metric=coverage)](https://sonarcloud.io/summary/new_code?id=bancolombia_devsecops-engine-tools) [![Python Version](https://img.shields.io/badge/python%20-%203.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20-blue)](#) +[![PyPI](https://img.shields.io/pypi/v/devsecops-engine-tools)](https://pypi.org/project/devsecops-engine-tools/) [![Docker Pulls](https://img.shields.io/docker/pulls/bancolombia/devsecops-engine-tools )](https://hub.docker.com/r/bancolombia/devsecops-engine-tools) @@ -39,7 +40,7 @@ pip3 install devsecops-engine-tools ### Scan running - flags (CLI) ```bash -devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container", "engine_risk", "engine_code"] --folder_path ["Folder path scan engine_iac, engine_code and engine_dependencies"] --platform ["k8s","cloudformation","docker", "openapi"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] --token_external_checks ["token_external_checks"] --xray_mode ["scan", "audit"] --image_to_scan ["image_to_scan"] +devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --remote_config_branch ["remote_config_branch"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container", "engine_risk", "engine_code"] --folder_path ["Folder path scan engine_iac, engine_code, engine_dependencies and engine_secret"] --platform ["k8s","cloudformation","docker", "openapi", "terraform"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] --token_external_checks ["token_external_checks"] --xray_mode ["scan", "audit"] --image_to_scan ["image_to_scan"] --dast_file_path ["dast_file_path"] ``` ### Structure Remote Config @@ -51,6 +52,9 @@ devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_con ┣ 📂engine_risk ┃ ┗ 📜ConfigTool.json ┃ ┗ 📜Exclusions.json + ┣ 📂engine_dast + ┃ ┗ 📜ConfigTool.json + ┃ ┗ 📜Exclusions.json ┣ 📂engine_sast ┃ ┗ 📂engine_iac ┃ ┗ 📜ConfigTool.json @@ -68,7 +72,7 @@ devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_con ┃ ┗ 📜ConfigTool.json ┃ ┗ 📜Exclusions.json ``` - +For more information visit [here](https://github.com/bancolombia/devsecops-engine-tools/blob/trunk/example_remote_config_local/README.md) #### Tools available for the modules (Configuration engine_core/ConfigTool.json) @@ -102,10 +106,14 @@ devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_con Free - ENGINE_SECRET + ENGINE_SECRET TRUFFLEHOG Free + + GITLEAKS + Free + ENGINE_CONTAINER PRISMA diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index a18fb4fb6..1bccd698e 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -58,6 +58,27 @@ Packages (directories) and modules (.py files) must be lowercase. +# Standard commits - Semantic release + +We use the semantic release library to manage the release in the project. Please validate at the time of contribution that it complies with the standard commits - and Pull Request based on the library definition: + +## [Semantic Release](https://semantic-release.gitbook.io/semantic-release) + +Available types: + - feat: A new feature + - fix: A bug fix + - docs: Documentation only changes + - style: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc) + - refactor: A code change that neither fixes a bug nor adds a feature + - perf: A code change that improves performance + - test: Adding missing tests or correcting existing tests + - build: Changes that affect the build system or external dependencies (example scopes: gulp, broccoli, npm) + - ci: Changes to our CI configuration files and scripts (example scopes: Travis, Circle, BrowserStack, SauceLabs) + - chore: Other changes that don't modify src or test files + - revert: Reverts a previous commit + +You can find out more here. [Semantic Versioning](https://semver.org/) + # GOVERNANCE Read more [Governance](https://github.com/bancolombia/devsecops-engine-tools/blob/trunk/docs/GOVERNANCE.md) \ No newline at end of file diff --git a/example_remote_config_local/README.md b/example_remote_config_local/README.md new file mode 100644 index 000000000..1a4c511ee --- /dev/null +++ b/example_remote_config_local/README.md @@ -0,0 +1,116 @@ +# Example use remote config for vulnerability management + +Initially, we need to know that the DevSecOps engine tools include a module for connecting to a vulnerability centralizer (DefectDojo). As a primary requirement for using this module, it must be considered whether a CMDB will be used or not. This is configurable through the remote config located in the [engine_core](https://github.com/bancolombia/devsecops-engine-tools/blob/trunk/example_remote_config_local/engine_core/ConfigTool.json). Below, examples of the two possible cases will be provided: + +### Using CMDB: +Let's suppose the CMDB response is as follows: +```json +{ + "count": 2, + "value": [ + { + "ApplicationCode": "code1-app", + "ApplicationName": "Example Application Name 1", + "ApplicationType": "Example Application type 1", + "ApplicationTag": "Example Application tag 1", + "ApplicationDescription": "Example Application description 1", + "Env": "PDN" + }, + { + "ApplicationCode": "code1-app", + "ApplicationName": "Example Application Name 1", + "ApplicationType": "Example Application type 1", + "ApplicationTag": "Example Application tag 1", + "ApplicationDescription": "Example Application description 1", + "Env": "DEV" + } + ] +} +``` + +Then, the remote config settings should look similar to this: +```json +"DEFECT_DOJO": { + "HOST_DEFECT_DOJO": "http://localhost:8080", + "LIMITS_QUERY": 100, + "MAX_RETRIES_QUERY": 5, + "CMDB": { + "USE_CMDB": true, + "HOST_CMDB": "http://host_cmdb_example", + "REGEX_EXPRESSION_CMDB": "^([^-]+)", + "CMDB_MAPPING_PATH": "/path/mapping_cmdb.json", + "CMDB_MAPPING": { + "PRODUCT_TYPE_NAME": "ApplicationType", + "PRODUCT_NAME": "ApplicationName", + "TAG_PRODUCT": "ApplicationTag", + "PRODUCT_DESCRIPTION": "ApplicationDescription", + "CODIGO_APP": "ApplicationCode" + }, + "CMDB_REQUEST_RESPONSE": { + "HEADERS": { + "Content-Type": "application/json", + "Api-Key": "tokenvalue" + }, + "METHOD": "GET", + "PARAMS": { + "appCode": "codappvalue" + }, + "RESPONSE": ["value", 0] + } + } +} +``` + +**Let’s detail the description for the elements under the CMDB key:** + +- *USE_CMDB:* The value is a boolean, indicating whether or not CMDB will be used. + +- *HOST_CMDB:* The URL of the API for querying the CMDB. + +- *REGEX_EXPRESSION_CMDB:* Regular expression. + +- *CMDB_MAPPING_PATH:* Location of the mapping for possible product types. + +- *CMDB_MAPPING:* Element containing the equivalent mappings between DefectDojo and the CMDB. + +- *CMDB_REQUEST_RESPONSE:* Contains the necessary elements to make a request to the CMDB. + +- *HEADERS:* Headers required to make the request. Note that the authentication type must be via Api-Key. The value "tokenvalue" should remain as is, as it is necessary for replacing the token. + +- *METHOD:* Can be either POST or GET. + +- *PARAMS:* Used if the selected METHOD is GET. The value "codappvalue" should remain as is, as it is necessary for replacement. + +- *BODY:* Used if the selected METHOD is POST. The value "codappvalue" should remain as is, as it is necessary for replacement. + +### Without Using CMDB: +For this case, three environment variables must be created according to the DevOps platform. +```bash +## Platform local +DET_VM_PRODUCT_TYPE_NAME="Example product type name" +DET_VM_PRODUCT_NAME="Example product type name" +DET_VM_PRODUCT_DESCRIPTION="Example product descrition" + +## Platform azure +VM_PRODUCT_TYPE_NAME="Example product type name" +VM_PRODUCT_NAME="Example product type name" +VM_PRODUCT_DESCRIPTION="Example product descrition" + +## Platform github +VM_PRODUCT_TYPE_NAME="Example product type name" +VM_PRODUCT_NAME="Example product type name" +VM_PRODUCT_DESCRIPTION="Example product descrition" +``` + +The remote config settings should look similar to this: +```json + "DEFECT_DOJO": { + "HOST_DEFECT_DOJO": "http://localhost:8080", + "LIMITS_QUERY": 100, + "MAX_RETRIES_QUERY": 5, + "CMDB": { + "USE_CMDB": false, + "REGEX_EXPRESSION_CMDB": "^([^-]+)", + } + } +``` \ No newline at end of file diff --git a/example_remote_config_local/engine_core/ConfigTool.json b/example_remote_config_local/engine_core/ConfigTool.json index c8783beb2..fb5515588 100644 --- a/example_remote_config_local/engine_core/ConfigTool.json +++ b/example_remote_config_local/engine_core/ConfigTool.json @@ -3,50 +3,94 @@ "SECRET_MANAGER": { "AWS": { "SECRET_NAME": "", + "USE_ROLE": false, "ROLE_ARN": "", "REGION_NAME": "" } }, "VULNERABILITY_MANAGER": { - "BRANCH_FILTER": "", + "BRANCH_FILTER": [ + "trunk", + "main" + ], "DEFECT_DOJO": { - "CMDB_MAPPING_PATH": "", - "HOST_CMDB": "", "HOST_DEFECT_DOJO": "", - "REGEX_EXPRESSION_CMDB": "", "LIMITS_QUERY": 100, - "MAX_RETRIES_QUERY": 5 + "MAX_RETRIES_QUERY": 5, + "CMDB": { + "USE_CMDB": false, + "HOST_CMDB": "", + "REGEX_EXPRESSION_CMDB": "", + "CMDB_MAPPING_PATH": "", + "CMDB_MAPPING": { + "PRODUCT_TYPE_NAME": "", + "PRODUCT_NAME": "", + "TAG_PRODUCT": "", + "PRODUCT_DESCRIPTION": "", + "CODIGO_APP": "" + }, + "CMDB_REQUEST_RESPONSE": { + "HEADERS": { + "Content-Type": "application/json", + "Api-Key": "tokenvalue" + }, + "METHOD": "GET|POST", + "PARAMS": { + "appCode": "codappvalue" + }, + "BODY": { + "appCode": "codappvalue" + }, + "RESPONSE": [0] + } + } } }, "METRICS_MANAGER": { "AWS": { "BUCKET": "", + "USE_ROLE": false, "ROLE_ARN": "", "REGION_NAME": "" } }, + "SBOM_MANAGER": { + "ENABLED": false, + "BRANCH_FILTER": [ + "trunk", + "main" + ], + "SYFT": { + "SYFT_VERSION": "1.17.0", + "OUTPUT_FORMAT": "cyclonedx-json" + } + }, "ENGINE_IAC": { - "ENABLED": "true", + "ENABLED": true, "TOOL": "CHECKOV|KUBESCAPE|KICS" }, "ENGINE_CONTAINER": { - "ENABLED": "true", + "ENABLED": true, "TOOL": "PRISMA|TRIVY" }, "ENGINE_DAST": { "ENABLED": "true", - "TOOL": "NUCLEI" + "TOOL": "NUCLEI", + "EXTRA_TOOLS": ["JWT"] }, "ENGINE_SECRET": { - "ENABLED": "true", - "TOOL": "TRUFFLEHOG" + "ENABLED": true, + "TOOL": "TRUFFLEHOG|GITLEAKS" }, "ENGINE_DEPENDENCIES": { - "ENABLED": "true", + "ENABLED": true, "TOOL": "XRAY|DEPENDENCY_CHECK" }, "ENGINE_CODE": { - "ENABLED": "true", + "ENABLED": true, "TOOL": "BEARER" + }, + "REPORT_SONAR": { + "ENABLED": true } } \ No newline at end of file diff --git a/example_remote_config_local/engine_dast/ConfigTool.json b/example_remote_config_local/engine_dast/ConfigTool.json new file mode 100644 index 000000000..a7a185d64 --- /dev/null +++ b/example_remote_config_local/engine_dast/ConfigTool.json @@ -0,0 +1,35 @@ +{ + "THRESHOLD": { + "VULNERABILITY": { + "Critical": 1, + "High": 8, + "Medium": 10, + "Low": 15 + }, + "COMPLIANCE": { + "Critical": 1 + } + }, + "MESSAGE_INFO_DAST": "If you have doubts, visit https://forum.example", + "NUCLEI": { + "VERSION": "3.3.5", + "DOWNLOAD_URL": "https://github.com/projectdiscovery/nuclei/releases/download/", + "USE_EXTERNAL_CHECKS_DIR": "True", + "EXTERNAL_DIR_OWNER": "dummie-username", + "EXTERNAL_DIR_REPOSITORY": "example-repo-templates", + "EXTERNAL_DIR_ASSET_NAME": "path/templates", + "EXTERNAL_CHECKS_PATH": "/local-templates" + }, + "JWT": { + "RULES": { + "JWT_ALGORITHM": { + "checkID": "ENGINE_JWT_001", + "environment": {"dev": "True", "pdn": "True", "qa": "True"}, + "guideline": "https://example.com/", + "severity": "Low", + "cvss": "", + "category": "Vulnerability" + } + } + } +} diff --git a/example_remote_config_local/engine_dast/Exclusions.json b/example_remote_config_local/engine_dast/Exclusions.json new file mode 100644 index 000000000..2a03a4192 --- /dev/null +++ b/example_remote_config_local/engine_dast/Exclusions.json @@ -0,0 +1,14 @@ +{ + "All": { + "JWT": [ + { + "id": "ENGINE_JWT_001", + "where": "all", + "create_date": "18112023", + "expired_date": "18032024", + "severity": "HIGH", + "hu": "4338704" + } + ] + } +} \ No newline at end of file diff --git a/example_remote_config_local/engine_risk/ConfigTool.json b/example_remote_config_local/engine_risk/ConfigTool.json index ed5d6b46f..a7df34801 100644 --- a/example_remote_config_local/engine_risk/ConfigTool.json +++ b/example_remote_config_local/engine_risk/ConfigTool.json @@ -3,9 +3,9 @@ "IGNORE_ANALYSIS_PATTERN": "(.*_test|test_.*)", "HANDLE_SERVICE_NAME": { "ENABLED": "false", - "ERASE_SERVICE_ENDING": [ - "_custom_ending1", - "_custom_ending2" + "CHECK_ENDING": [ + "_ending1", + "_ending2" ], "REGEX_GET_SERVICE_CODE": "[^_]+", "REGEX_GET_WORDS": "[_-]", @@ -42,15 +42,25 @@ "engine_dependencies": 0 }, "age": 0.0333, + "max_age": 12, "epss_score": 100 }, + "TAG_EXCLUSION_DAYS": { + "tag1": 10, + "tag2": 20 + }, + "TAG_BLACKLIST": [ + "tag1", + "tag2" + ], "THRESHOLD": { - "REMEDIATION_RATE": 50, + "REMEDIATION_RATE": { + "1": 0, + "5": 30, + "10": 50, + "other": 70 + }, "RISK_SCORE": 10, - "TAG_BLACKLIST": [ - "tag1", - "tag2" - ], "TAG_MAX_AGE": 999 } } \ No newline at end of file diff --git a/example_remote_config_local/engine_sast/engine_iac/ConfigTool.json b/example_remote_config_local/engine_sast/engine_iac/ConfigTool.json old mode 100644 new mode 100755 index 3d037bc7c..6a0cf25b3 --- a/example_remote_config_local/engine_sast/engine_iac/ConfigTool.json +++ b/example_remote_config_local/engine_sast/engine_iac/ConfigTool.json @@ -15,16 +15,20 @@ "Critical": 1 } }, - "UPDATE_SERVICE_WITH_FILE_NAME_CFT": "False", + "UPDATE_SERVICE_WITH_FILE_NAME_CFT": false, "CHECKOV": { "VERSION": "2.3.296", - "USE_EXTERNAL_CHECKS_GIT": "False", + "USE_EXTERNAL_CHECKS_GIT": false, "EXTERNAL_CHECKS_GIT": "", "EXTERNAL_GIT_SSH_HOST": "", "EXTERNAL_GIT_PUBLIC_KEY_FINGERPRINT": "", - "USE_EXTERNAL_CHECKS_DIR": "False", + "USE_EXTERNAL_CHECKS_DIR": false, "EXTERNAL_DIR_OWNER": "", "EXTERNAL_DIR_REPOSITORY": "", + "APP_ID_GITHUB":"", + "INSTALLATION_ID_GITHUB":"", + "DEFAULT_SEVERITY": "Critical", + "DEFAULT_CATEGORY": "Compliance", "RULES": { "RULES_DOCKER": { "CKV_DOCKER_1": { @@ -80,6 +84,20 @@ "cvss": "", "category": "Vulnerability" } + }, + "RULES_TERRAFORM": { + "CKV_AWS_144": { + "checkID": "IAC-CKV-TERRAFORM-1 Ensure terraform", + "environment": { + "dev": true, + "pdn": true, + "qa": true + }, + "guideline": "guideline", + "severity": "Medium", + "cvss": "", + "category": "Vulnerability" + } } } }, diff --git a/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json b/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json old mode 100644 new mode 100755 index ac4585298..0046b5531 --- a/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json +++ b/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json @@ -1,7 +1,5 @@ { - "IGNORE_SEARCH_PATTERN": [ - "test" - ], + "IGNORE_SEARCH_PATTERN": "(.*test.*)", "MESSAGE_INFO_ENGINE_SECRET": "message custom", "THRESHOLD": { "VULNERABILITY": { @@ -16,10 +14,34 @@ }, "TARGET_BRANCHES": ["trunk", "develop"], "trufflehog": { + "VERSION": "1.2.3", "EXCLUDE_PATH": [".git"], "NUMBER_THREADS": 4, - "ENABLE_CUSTOM_RULES" : "True", + "ENABLE_CUSTOM_RULES" : true, "EXTERNAL_DIR_OWNER": "ExternalOrg", - "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks" + "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks", + "APP_ID_GITHUB":"", + "INSTALLATION_ID_GITHUB":"", + "USE_EXTERNAL_CHECKS_GIT": false, + "USE_EXTERNAL_CHECKS_DIR": true, + "RULES": { + "MISCONFIGURATION_SCANNING": { + "References": "https://reference.url/", + "Mitigation": "Make sure you only enable the Spring Boot Actuator endpoints that you really need and restrict access to these endpoints." + } + } + }, + "gitleaks": { + "VERSION": "8.21.1", + "EXCLUDE_PATH": [".git"], + "NUMBER_THREADS": 4, + "ALLOW_IGNORE_LEAKS": false, + "ENABLE_CUSTOM_RULES" : true, + "EXTERNAL_DIR_OWNER": "ExternalOrg", + "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks", + "APP_ID_GITHUB":"", + "INSTALLATION_ID_GITHUB":"", + "USE_EXTERNAL_CHECKS_GIT": false, + "USE_EXTERNAL_CHECKS_DIR": true } } \ No newline at end of file diff --git a/example_remote_config_local/engine_sca/engine_container/ConfigTool.json b/example_remote_config_local/engine_sca/engine_container/ConfigTool.json old mode 100644 new mode 100755 index 7c69ff070..372e8a4a2 --- a/example_remote_config_local/engine_sca/engine_container/ConfigTool.json +++ b/example_remote_config_local/engine_sca/engine_container/ConfigTool.json @@ -3,11 +3,21 @@ "TWISTCLI_PATH": "twistcli", "PRISMA_CONSOLE_URL": "", "PRISMA_ACCESS_KEY": "", - "PRISMA_API_VERSION":"" + "PRISMA_API_VERSION":"", + "SBOM_FORMAT": "cyclonedx_json" }, "TRIVY": { - "TRIVY_VERSION": "0.51.4" + "TRIVY_VERSION": "0.51.4", + "SBOM_FORMAT": "cyclonedx" }, + "SBOM": { + "ENABLED": false, + "BRANCH_FILTER": [ + "trunk", + "main" + ] + }, + "GET_IMAGE_BASE": true, "MESSAGE_INFO_ENGINE_CONTAINER": "message custom", "IGNORE_SEARCH_PATTERN":"(.*_demo0|.*_cer)", "THRESHOLD": { @@ -17,13 +27,36 @@ "Medium": 20, "Low": 999 }, - "CUSTOM_VULNERABILITY": { - "PATTERN_APPS": "^(?!(App1|Apptest)$).*(App2.*|.*App3.*|.*App.*)", - "VULNERABILITY": { + "QUALITY_VULNERABILITY_MANAGEMENT": { + "PTS": [ + { + "Product Type Name": { + "APPS": [ + "CodeApp", + "CodeApp1", + "CodeApp12" + ], + "PROFILE": "STRONG" + } + }, + { + "Product Type Name2": { + "APPS": "ALL", + "PROFILE": "MODERATE" + } + } + ], + "STRONG": { "Critical": 0, "High": 0, "Medium": 5, - "Low": 10 + "Low": 15 + }, + "MODERATE": { + "Critical": 1, + "High": 3, + "Medium": 5, + "Low": 15 } }, "COMPLIANCE": { diff --git a/example_remote_config_local/engine_sca/engine_dependencies/ConfigTool.json b/example_remote_config_local/engine_sca/engine_dependencies/ConfigTool.json index 893ea3a54..4d0d6b93f 100644 --- a/example_remote_config_local/engine_sca/engine_dependencies/ConfigTool.json +++ b/example_remote_config_local/engine_sca/engine_dependencies/ConfigTool.json @@ -2,7 +2,10 @@ "XRAY": { "CLI_VERSION": "2.55.0", "REGEX_EXPRESSION_EXTENSIONS": "\\.(jar|ear|war)$", - "PACKAGES_TO_SCAN": ["node_modules", "site-packages"] + "PACKAGES_TO_SCAN": ["node_modules", "site-packages"], + "STDERR_EXPECTED_WORDS": ["Technology", "WorkingDirectory", "Descriptors"], + "STDERR_BREAK_ERRORS": ["NoSuchFileException"], + "STDERR_ACCEPTED_ERRORS": ["What went wrong", "Caused by"] }, "IGNORE_ANALYSIS_PATTERN": "(.*_test)", "MESSAGE_INFO_ENGINE_DEPENDENCIES": "message custom", @@ -19,8 +22,9 @@ "CVE": ["CVE-123123"] }, "DEPENDENCY_CHECK": { - "CLI_VERSION": "10.0.4", + "CLI_VERSION": "11.1.0", "REGEX_EXPRESSION_EXTENSIONS": "\\.(jar|ear|war)$", - "PACKAGES_TO_SCAN": ["node_modules", "site-packages"] + "PACKAGES_TO_SCAN": ["node_modules", "site-packages"], + "VULNERABILITY_CONFIDENCE" : ["highest"] } } \ No newline at end of file diff --git a/example_remote_config_local/report_sonar/ConfigTool.json b/example_remote_config_local/report_sonar/ConfigTool.json new file mode 100644 index 000000000..7fdf3e7b3 --- /dev/null +++ b/example_remote_config_local/report_sonar/ConfigTool.json @@ -0,0 +1,13 @@ +{ + "IGNORE_SEARCH_PATTERN": ".*test.*", + "TARGET_BRANCHES": ["trunk", "develop", "master"], + "PIPELINE_COMPONENTS": { + "EXAMPLE_MULTICOMPONENT_PIPELINE": [ + "component1", + "component2", + "component3", + "component4", + "component5" + ] + } +} diff --git a/ide_extension/intellij/build.gradle.kts b/ide_extension/intellij/build.gradle.kts index ffb6ec3c0..db2db981f 100644 --- a/ide_extension/intellij/build.gradle.kts +++ b/ide_extension/intellij/build.gradle.kts @@ -9,19 +9,19 @@ plugins { id("java") id("jacoco") // IntelliJ Platform Gradle Plugin - id("org.jetbrains.intellij.platform") version "2.1.0" + id("org.jetbrains.intellij.platform") version "2.2.1" // Gradle Changelog Plugin id("org.jetbrains.changelog") version "2.2.1" // Gradle Sonar Plugin - id("org.sonarqube") version "5.1.0.4882" + id("org.sonarqube") version "6.0.1.5171" } group = properties("pluginGroup").get() version = properties("pluginVersion").get() java { - sourceCompatibility = JavaVersion.VERSION_17 - targetCompatibility = JavaVersion.VERSION_17 + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 } repositories { @@ -44,22 +44,21 @@ dependencies { // Plugin Dependencies. Uses `platformPlugins` property from the gradle.properties file for plugin from JetBrains Marketplace. plugins(properties("platformPlugins").map { it.split(',') }) - instrumentationTools() pluginVerifier() zipSigner() testFramework(TestFrameworkType.Platform) } implementation("com.squareup.okhttp3:okhttp") - implementation("org.jetbrains:annotations:25.0.0") + implementation("org.jetbrains:annotations:26.0.1") - compileOnly("org.projectlombok:lombok:1.18.34") - annotationProcessor("org.projectlombok:lombok:1.18.34") + compileOnly("org.projectlombok:lombok:1.18.36") + annotationProcessor("org.projectlombok:lombok:1.18.36") - testCompileOnly("org.projectlombok:lombok:1.18.34") - testAnnotationProcessor("org.projectlombok:lombok:1.18.34") + testCompileOnly("org.projectlombok:lombok:1.18.36") + testAnnotationProcessor("org.projectlombok:lombok:1.18.36") - testImplementation("org.mockito:mockito-core:5.14.1") + testImplementation("org.mockito:mockito-core:5.15.2") implementation(platform("com.squareup.okhttp3:okhttp-bom:4.12.0")) } diff --git a/ide_extension/intellij/gradle.properties b/ide_extension/intellij/gradle.properties index 18238667f..ba8e1954e 100644 --- a/ide_extension/intellij/gradle.properties +++ b/ide_extension/intellij/gradle.properties @@ -6,23 +6,23 @@ pluginDescription=DevSecOps Engine Tools plugin which allows you to execute DevS pluginRepositoryUrl=https://github.com/bancolombia/devsecops-engine-tools pluginVendorName=Bancolombia # SemVer format -> https://semver.org -pluginVersion=1.3.0 +pluginVersion=1.7.0 # Supported build number ranges and IntelliJ Platform versions -> https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html -pluginSinceBuild=241 -pluginUntilBuild=242.* +pluginSinceBuild=243 +pluginUntilBuild=251.* # IntelliJ Platform Properties -> https://plugins.jetbrains.com/docs/intellij/tools-gradle-intellij-plugin.html#configuration-intellij-extension platformType=IC -platformVersion=2024.2.3 +platformVersion=2024.3.1.1 # Plugin Dependencies -> https://plugins.jetbrains.com/docs/intellij/plugin-dependencies.html # Example: platformPlugins = com.jetbrains.php:203.4449.22, org.intellij.scala:2023.3.27@EAP platformPlugins= # Example: platformBundledPlugins = com.intellij.java platformBundledPlugins= # Gradle Releases -> https://github.com/gradle/gradle/releases -gradleVersion=8.10.2 +gradleVersion=8.12 # Opt-out flag for bundling Kotlin standard library -> https://jb.gg/intellij-platform-kotlin-stdlib kotlin.stdlib.default.dependency=false # Enable Gradle Configuration Cache -> https://docs.gradle.org/current/userguide/configuration_cache.html -org.gradle.configuration-cache=true +org.gradle.configuration-cache=false # Enable Gradle Build Cache -> https://docs.gradle.org/current/userguide/build_cache.html -org.gradle.caching=true \ No newline at end of file +org.gradle.caching=false \ No newline at end of file diff --git a/ide_extension/intellij/gradle/wrapper/gradle-wrapper.jar b/ide_extension/intellij/gradle/wrapper/gradle-wrapper.jar index 2c3521197..a4b76b953 100644 Binary files a/ide_extension/intellij/gradle/wrapper/gradle-wrapper.jar and b/ide_extension/intellij/gradle/wrapper/gradle-wrapper.jar differ diff --git a/ide_extension/intellij/gradle/wrapper/gradle-wrapper.properties b/ide_extension/intellij/gradle/wrapper/gradle-wrapper.properties index df97d72b8..cea7a793a 100644 --- a/ide_extension/intellij/gradle/wrapper/gradle-wrapper.properties +++ b/ide_extension/intellij/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/ide_extension/intellij/src/main/java/co/com/bancolombia/devsecopsenginetools/ui/configuration/ProjectConfiguration.java b/ide_extension/intellij/src/main/java/co/com/bancolombia/devsecopsenginetools/ui/configuration/ProjectConfiguration.java index 14fc41471..dfa00f093 100644 --- a/ide_extension/intellij/src/main/java/co/com/bancolombia/devsecopsenginetools/ui/configuration/ProjectConfiguration.java +++ b/ide_extension/intellij/src/main/java/co/com/bancolombia/devsecopsenginetools/ui/configuration/ProjectConfiguration.java @@ -135,16 +135,14 @@ private void loadConfig() { variableGroups.setText(projectSettings.getAzureDevOpsVariableGroups()); releaseDefinition.setText(projectSettings.getAzureReleaseDefinitionId()); stageName.setText(projectSettings.getAzureReleaseStageName()); - - iacDirectory.addBrowseFolderListener("Select IaC Resources", "Select IaC resources directory", - project, FileChooserDescriptorFactory.createMultipleFoldersDescriptor()); - dotEnvVariables.addBrowseFolderListener("Select .env File", "Select .env file", - project, FileChooserDescriptorFactory.createSingleFileDescriptor()); - - dockerFilePath.addBrowseFolderListener("Select Dockerfile", "Select Dockerfile resource", - project, FileChooserDescriptorFactory.createSingleFileDescriptor()); - buildContextPath.addBrowseFolderListener("Select Build Context Path", "Select image build context", - project, FileChooserDescriptorFactory.createSingleFolderDescriptor()); + iacDirectory.addBrowseFolderListener(project, FileChooserDescriptorFactory.createMultipleFoldersDescriptor() + .withTitle("Select IaC Resources").withDescription("Select IaC resources directory")); + dotEnvVariables.addBrowseFolderListener(project, FileChooserDescriptorFactory.createSingleFileDescriptor() + .withTitle("Select .env File").withDescription("Select .env file")); + dockerFilePath.addBrowseFolderListener(project, FileChooserDescriptorFactory.createSingleFileDescriptor() + .withTitle("Select Dockerfile").withDescription("Select Dockerfile resource")); + buildContextPath.addBrowseFolderListener(project, FileChooserDescriptorFactory.createSingleFolderDescriptor() + .withTitle("Select Build Context Path").withDescription("Select image build context")); } diff --git a/tools/README.md b/tools/README.md index 8eb60e8b6..7b505711e 100644 --- a/tools/README.md +++ b/tools/README.md @@ -106,8 +106,11 @@ devsecops_engine_tools ├───engine_utilities -> Utilities transversal. | azuredevops | defect_dojo +| git_cli | github | input_validations +| sbom +| sonarqube | ssh | utils ``` diff --git a/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py b/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py old mode 100644 new mode 100755 index 2e84cf330..b858b0e9b --- a/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py +++ b/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py @@ -22,6 +22,7 @@ from devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.printer_pretty_table.printer_pretty_table import ( PrinterPrettyTable, ) +from devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft import Syft import sys import argparse from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger @@ -69,7 +70,15 @@ def get_inputs_from_cli(args): "--remote_config_repo", type=str, required=True, - help="Name or Folder Path of Config Repo", + help="Name or Folder Path of Remote Config Repo", + ) + parser.add_argument( + "-rcb", + "--remote_config_branch", + type=str, + required=False, + default="", + help="Name of the branch of Remote Config Repo", ) parser.add_argument( "-t", @@ -92,12 +101,12 @@ def get_inputs_from_cli(args): "--folder_path", type=str, required=False, - help="Folder Path to scan, only apply engine_iac, engine_code and engine_dependencies tools", + help="Folder Path to scan, only apply engine_iac, engine_code, engine_secret and engine_dependencies tools", ) parser.add_argument( "-p", "--platform", - type=parse_choices({"all", "docker", "k8s", "cloudformation", "openapi"}), + type=parse_choices({"all", "docker", "k8s", "cloudformation", "openapi", "terraform"}), required=False, default="all", help="Platform to scan, only apply engine_iac tool", @@ -107,6 +116,7 @@ def get_inputs_from_cli(args): choices=["true", "false"], type=str, required=False, + default="false", help="Use Secrets Manager to get the tokens", ) parser.add_argument( @@ -114,6 +124,7 @@ def get_inputs_from_cli(args): choices=["true", "false"], type=str, required=False, + default="false", help="Use Vulnerability Management to send the vulnerabilities to the platform", ) parser.add_argument( @@ -121,6 +132,7 @@ def get_inputs_from_cli(args): choices=["true", "false"], type=str, required=False, + default="false", help="Enable or Disable the send metrics to the driven adapter metrics", ) parser.add_argument( @@ -144,7 +156,7 @@ def get_inputs_from_cli(args): parser.add_argument( "--token_external_checks", required=False, - help="Token for downloading external checks from engine_iac or engine_secret if is necessary. Ej: github:token, ssh:privatekey:pass", + help="Token for downloading external checks from engine_iac or engine_secret if is necessary. Ej: github_token:token, github_app:private_key, ssh:privatekey:pass", ) parser.add_argument( "--xray_mode", @@ -158,10 +170,17 @@ def get_inputs_from_cli(args): required=False, help="Name of image to scan for engine_container", ) + parser.add_argument( + "--dast_file_path", + required=False, + help="File path containing the configuration, structured according to the documentation, \ + for the API or web application to be scanned by the DAST tool." + ) args = parser.parse_args() return { "platform_devops": args.platform_devops, "remote_config_repo": args.remote_config_repo, + "remote_config_branch": args.remote_config_branch, "tool": args.tool, "folder_path": args.folder_path, "platform": args.platform, @@ -174,7 +193,8 @@ def get_inputs_from_cli(args): "token_engine_dependencies": args.token_engine_dependencies, "token_external_checks": args.token_external_checks, "xray_mode": args.xray_mode, - "image_to_scan": args.image_to_scan + "image_to_scan": args.image_to_scan, + "dast_file_path": args.dast_file_path } @@ -191,8 +211,9 @@ def application_core(): "github": GithubActions(), "local": RuntimeLocal(), }.get(args["platform_devops"]) - printer_table_gateway = PrinterPrettyTable() metrics_manager_gateway = S3Manager() + printer_table_gateway = PrinterPrettyTable() + sbom_tool_gateway = Syft() init_engine_core( vulnerability_management_gateway, @@ -200,6 +221,7 @@ def application_core(): devops_platform_gateway, printer_table_gateway, metrics_manager_gateway, + sbom_tool_gateway, args, ) except Exception as e: diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/model/component.py b/tools/devsecops_engine_tools/engine_core/src/domain/model/component.py new file mode 100644 index 000000000..4944dfd21 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_core/src/domain/model/component.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass + + +@dataclass +class Component: + name: str + version: str \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/model/custom_level_vulnerability.py b/tools/devsecops_engine_tools/engine_core/src/domain/model/custom_level_vulnerability.py deleted file mode 100644 index 37cce5d2c..000000000 --- a/tools/devsecops_engine_tools/engine_core/src/domain/model/custom_level_vulnerability.py +++ /dev/null @@ -1,8 +0,0 @@ -from devsecops_engine_tools.engine_core.src.domain.model.level_vulnerability import ( - LevelVulnerability, -) - -class CustomLevelVulnerability: - def __init__(self, data): - self.pattern_apps = data.get("PATTERN_APPS") - self.vulnerability = LevelVulnerability(data.get("VULNERABILITY")) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/model/exclusions.py b/tools/devsecops_engine_tools/engine_core/src/domain/model/exclusions.py index dfa4bddf7..952492d7c 100644 --- a/tools/devsecops_engine_tools/engine_core/src/domain/model/exclusions.py +++ b/tools/devsecops_engine_tools/engine_core/src/domain/model/exclusions.py @@ -5,10 +5,15 @@ class Exclusions: def __init__(self, **kwargs): self.id = kwargs.get("id", "") - self.where = kwargs.get("where", "") + self.where = kwargs.get("where", "all") self.cve_id = kwargs.get("cve_id", "") self.create_date = kwargs.get("create_date", "") self.expired_date = kwargs.get("expired_date", "") self.severity = kwargs.get("severity", "") self.hu = kwargs.get("hu", "") - self.reason = kwargs.get("reason", "Risk acceptance") + self.reason = kwargs.get("reason", "Risk Accepted") + self.vm_id = kwargs.get("vm_id", "") + self.vm_id_url = kwargs.get("vm_id_url", "") + self.service = kwargs.get("service", "") + self.tags = kwargs.get("tags", []) + self.check_in_desc = kwargs.get("x86.image.name", []) diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/devops_platform_gateway.py b/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/devops_platform_gateway.py index e5d83e167..8b8f9cd33 100644 --- a/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/devops_platform_gateway.py +++ b/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/devops_platform_gateway.py @@ -3,7 +3,7 @@ class DevopsPlatformGateway(metaclass=ABCMeta): @abstractmethod - def get_remote_config(self, repository, path): + def get_remote_config(self, repository, path, branch): "get_remote_config" @abstractmethod diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/sbom_manager.py b/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/sbom_manager.py new file mode 100644 index 000000000..7d5480cf8 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/sbom_manager.py @@ -0,0 +1,11 @@ +from abc import ABCMeta, abstractmethod +from devsecops_engine_tools.engine_core.src.domain.model.component import ( + Component, +) + +class SbomManagerGateway(metaclass=ABCMeta): + @abstractmethod + def get_components( + self, artifact, config, service_name + ) -> "list[Component]": + "get_components" diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/vulnerability_management_gateway.py b/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/vulnerability_management_gateway.py index 6cce8b900..bc26abfd0 100644 --- a/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/vulnerability_management_gateway.py +++ b/tools/devsecops_engine_tools/engine_core/src/domain/model/gateway/vulnerability_management_gateway.py @@ -1,7 +1,7 @@ from abc import ABCMeta, abstractmethod from devsecops_engine_tools.engine_core.src.domain.model.vulnerability_management import VulnerabilityManagement - +from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import DevopsPlatformGateway class VulnerabilityManagementGateway(metaclass=ABCMeta): @abstractmethod @@ -10,6 +10,10 @@ def send_vulnerability_management( ): "send_vulnerability_management" + @abstractmethod + def get_product_type_service(self, service, dict_args, secret_tool, config_tool): + "get_product_type_service" + @abstractmethod def get_findings_excepted( self, service, dict_args, secret_tool, config_tool @@ -27,3 +31,9 @@ def get_active_engagements( self, engagement_name, dict_args, secret_tool, config_tool ): "get_active_engagements" + + @abstractmethod + def send_sbom_components( + self, sbom_components, service, dict_args, secret_tool, config_tool + ): + "send_sbom_components" diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/model/report.py b/tools/devsecops_engine_tools/engine_core/src/domain/model/report.py index 872a60e0a..1cf2519fb 100644 --- a/tools/devsecops_engine_tools/engine_core/src/domain/model/report.py +++ b/tools/devsecops_engine_tools/engine_core/src/domain/model/report.py @@ -4,6 +4,8 @@ @dataclass class Report: def __init__(self, **kwargs): + self.vm_id = kwargs.get("vm_id", "") + self.vm_id_url = kwargs.get("vm_id_url", "") self.id = kwargs.get("id", []) self.vuln_id_from_tool = kwargs.get("vuln_id_from_tool", "") self.where = kwargs.get("where", "") @@ -26,9 +28,12 @@ def __init__(self, **kwargs): self.vul_description = kwargs.get("vul_description", "") self.risk_accepted = kwargs.get("risk_accepted", "") self.false_p = kwargs.get("false_p", "") + self.out_of_scope = kwargs.get("out_of_scope", "") self.service = kwargs.get("service", "") self.reason = kwargs.get("reason", "") self.component_name = kwargs.get("component_name", "") self.component_version = kwargs.get("component_version", "") self.file_path = kwargs.get("file_path", "") - self.endpoints = kwargs.get("endpoints", "") \ No newline at end of file + self.endpoints = kwargs.get("endpoints", "") + self.unique_id_from_tool = kwargs.get("unique_id_from_tool", "") + self.out_of_scope = kwargs.get("out_of_scope", "") \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/model/threshold.py b/tools/devsecops_engine_tools/engine_core/src/domain/model/threshold.py index 8728475e1..4e0bae630 100644 --- a/tools/devsecops_engine_tools/engine_core/src/domain/model/threshold.py +++ b/tools/devsecops_engine_tools/engine_core/src/domain/model/threshold.py @@ -4,13 +4,10 @@ from devsecops_engine_tools.engine_core.src.domain.model.level_compliance import ( LevelCompliance, ) -from devsecops_engine_tools.engine_core.src.domain.model.custom_level_vulnerability import ( - CustomLevelVulnerability, -) class Threshold: def __init__(self, data): self.vulnerability = LevelVulnerability(data.get("VULNERABILITY")) self.compliance = LevelCompliance(data.get("COMPLIANCE")) self.cve = data.get("CVE",[]) - self.custom_vulnerability = CustomLevelVulnerability(data.get("CUSTOM_VULNERABILITY")) if data.get("CUSTOM_VULNERABILITY") else None + self.quality_vulnerability_management = data.get("QUALITY_VULNERABILITY_MANAGEMENT") if data.get("QUALITY_VULNERABILITY_MANAGEMENT") else None diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/model/vulnerability_management.py b/tools/devsecops_engine_tools/engine_core/src/domain/model/vulnerability_management.py index 1d7c147e9..c4630abbe 100644 --- a/tools/devsecops_engine_tools/engine_core/src/domain/model/vulnerability_management.py +++ b/tools/devsecops_engine_tools/engine_core/src/domain/model/vulnerability_management.py @@ -18,3 +18,6 @@ class VulnerabilityManagement: branch_tag: str commit_hash: str environment: str + vm_product_type_name: str + vm_product_name: str + vm_product_description: str diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/usecases/break_build.py b/tools/devsecops_engine_tools/engine_core/src/domain/usecases/break_build.py index f6509f566..89b7baebf 100644 --- a/tools/devsecops_engine_tools/engine_core/src/domain/usecases/break_build.py +++ b/tools/devsecops_engine_tools/engine_core/src/domain/usecases/break_build.py @@ -67,9 +67,6 @@ def process(self, findings_list: "list[Finding]", input_core: InputCore, args: a "compliances": {}, } - if threshold.custom_vulnerability and bool(re.match(threshold.custom_vulnerability.pattern_apps, input_core.scope_pipeline, re.IGNORECASE)): - threshold.vulnerability = threshold.custom_vulnerability.vulnerability - if len(findings_list) != 0: self._apply_policie_exception_new_vulnerability_industry( findings_list, exclusions, args diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/usecases/handle_risk.py b/tools/devsecops_engine_tools/engine_core/src/domain/usecases/handle_risk.py index 043e77e71..01ec28842 100644 --- a/tools/devsecops_engine_tools/engine_core/src/domain/usecases/handle_risk.py +++ b/tools/devsecops_engine_tools/engine_core/src/domain/usecases/handle_risk.py @@ -51,7 +51,7 @@ def _get_all_from_vm(self, dict_args, secret_tool, remote_config, service): "Error getting finding list in handle risk: {0}".format(str(e)) ) - def _filter_engagements(self, engagements, service, risk_config): + def _filter_engagements(self, engagements, service, initial_services, risk_config): filtered_engagements = [] min_word_length = risk_config["HANDLE_SERVICE_NAME"]["MIN_WORD_LENGTH"] words = [ @@ -63,67 +63,125 @@ def _filter_engagements(self, engagements, service, risk_config): ] check_words_regex = risk_config["HANDLE_SERVICE_NAME"]["REGEX_CHECK_WORDS"] min_word_amount = risk_config["HANDLE_SERVICE_NAME"]["MIN_WORD_AMOUNT"] + endings = risk_config["HANDLE_SERVICE_NAME"]["CHECK_ENDING"] + + initial_services_lower = [service.lower() for service in initial_services] + for engagement in engagements: - if service.lower() in engagement.name.lower(): - filtered_engagements += [engagement.name] + if engagement.name.lower() in initial_services_lower: + filtered_engagements += [engagement] elif re.search(check_words_regex, engagement.name.lower()) and ( sum(1 for word in words if word.lower() in engagement.name.lower()) >= min_word_amount ): - filtered_engagements += [engagement.name] + filtered_engagements += [engagement] + elif endings: + if any( + (service.lower() + ending.lower() == engagement.name.lower()) + for ending in endings + ): + filtered_engagements += [engagement] + return filtered_engagements def _exclude_services(self, dict_args, pipeline_name, service_list): risk_exclusions = self.devops_platform_gateway.get_remote_config( - dict_args["remote_config_repo"], "engine_risk/Exclusions.json" + dict_args["remote_config_repo"], "engine_risk/Exclusions.json", dict_args["remote_config_branch"] ) if ( pipeline_name in risk_exclusions and risk_exclusions[pipeline_name].get("SKIP_SERVICE", 0) and risk_exclusions[pipeline_name]["SKIP_SERVICE"].get("services", 0) ): - services_to_exclude = risk_exclusions[pipeline_name]["SKIP_SERVICE"].get( - "services", [] + services_to_exclude = set( + risk_exclusions[pipeline_name]["SKIP_SERVICE"].get("services", []) ) - service_excluded = [] - for service in service_list: - if service in services_to_exclude: - service_list.remove(service) - service_excluded += [service] - print(f"Services to exclude: {service_excluded}") - logger.info(f"Services to exclude: {service_excluded}") + + remaining_engagements = [ + engagement + for engagement in service_list + if engagement.name.lower() + not in [service.lower() for service in services_to_exclude] + ] + excluded_engagements = [ + engagement + for engagement in service_list + if engagement.name.lower() + in [service.lower() for service in services_to_exclude] + ] + + print( + f"Services to exclude: {[engagement.name for engagement in excluded_engagements]}" + ) + logger.info( + f"Services to exclude: {[engagement.name for engagement in excluded_engagements]}" + ) + + return remaining_engagements return service_list + def _should_skip_analysis(self, remote_config, pipeline_name, exclusions): + ignore_pattern = remote_config["IGNORE_ANALYSIS_PATTERN"] + return re.match(ignore_pattern, pipeline_name, re.IGNORECASE) or ( + pipeline_name in exclusions + and exclusions[pipeline_name].get("SKIP_TOOL", 0) + ) + def process(self, dict_args: any, remote_config: any): + risk_config = self.devops_platform_gateway.get_remote_config( + dict_args["remote_config_repo"], "engine_risk/ConfigTool.json", dict_args["remote_config_branch"] + ) + risk_exclusions = self.devops_platform_gateway.get_remote_config( + dict_args["remote_config_repo"], "engine_risk/Exclusions.json", dict_args["remote_config_branch"] + ) + pipeline_name = self.devops_platform_gateway.get_variable("pipeline_name") + + input_core = InputCore( + [], + {}, + "", + "", + pipeline_name, + self.devops_platform_gateway.get_variable("stage").capitalize(), + ) + + if self._should_skip_analysis(risk_config, pipeline_name, risk_exclusions): + print("Tool skipped by DevSecOps Policy.") + dict_args["send_metrics"] = "false" + return [], input_core + secret_tool = None if dict_args["use_secrets_manager"] == "true": secret_tool = self.secrets_manager_gateway.get_secret(remote_config) - risk_config = self.devops_platform_gateway.get_remote_config( - dict_args["remote_config_repo"], "engine_risk/ConfigTool.json" - ) - - pipeline_name = self.devops_platform_gateway.get_variable("pipeline_name") service = pipeline_name service_list = [] + initial_services = [] + initial_services += [service] + + match_parent = re.match( + risk_config["PARENT_ANALYSIS"]["REGEX_GET_PARENT"], service + ) + if risk_config["PARENT_ANALYSIS"]["ENABLED"].lower() == "true" and match_parent: + parent_service = match_parent.group(0) + initial_services += [parent_service] if risk_config["HANDLE_SERVICE_NAME"]["ENABLED"].lower() == "true": service = next( ( pipeline_name.replace(ending, "") - for ending in risk_config["HANDLE_SERVICE_NAME"][ - "ERASE_SERVICE_ENDING" - ] + for ending in risk_config["HANDLE_SERVICE_NAME"]["CHECK_ENDING"] if pipeline_name.endswith(ending) ), pipeline_name, ) + initial_services += [service] match_service_code = re.match( risk_config["HANDLE_SERVICE_NAME"]["REGEX_GET_SERVICE_CODE"], service ) if match_service_code: service_code = match_service_code.group(0) - service_list += [ + initial_services += [ service.format(service_code=service_code) for service in risk_config["HANDLE_SERVICE_NAME"]["ADD_SERVICES"] ] @@ -131,31 +189,33 @@ def process(self, dict_args: any, remote_config: any): service_code, dict_args, secret_tool, remote_config ) service_list += self._filter_engagements( - engagements, service, risk_config + engagements, service, initial_services, risk_config ) + else: + for service in initial_services: + engagements = self.vulnerability_management.get_active_engagements( + service, dict_args, secret_tool, remote_config + ) + for engagement in engagements: + if engagement.name.lower() == service.lower(): + service_list += [engagement] + break - service_list += [service] - - match_parent = re.match( - risk_config["PARENT_ANALYSIS"]["REGEX_GET_PARENT"], service - ) - if risk_config["PARENT_ANALYSIS"]["ENABLED"].lower() == "true" and match_parent: - parent_service = match_parent.group(0) - service_list += [parent_service] - - service_list = list(set(service_list)) new_service_list = self._exclude_services( dict_args, pipeline_name, service_list ) - print(f"Services to analyze: {new_service_list}") - logger.info(f"Services to analyze: {new_service_list}") + for engagement in new_service_list: + print(f"Service to analyze: {engagement.name}, URL: {engagement.vm_url}") + logger.info( + f"Service to analyze: {engagement.name}, URL: {engagement.vm_url}" + ) findings = [] exclusions = [] for service in new_service_list: findings_list, exclusions_list = self._get_all_from_vm( - dict_args, secret_tool, remote_config, service + dict_args, secret_tool, remote_config, service.name ) findings += findings_list exclusions += exclusions_list @@ -164,15 +224,9 @@ def process(self, dict_args: any, remote_config: any): dict_args, findings, exclusions, + [service.name for service in new_service_list], self.devops_platform_gateway, self.print_table_gateway, ) - input_core = InputCore( - [], - {}, - "", - "", - pipeline_name, - self.devops_platform_gateway.get_variable("stage").capitalize(), - ) + return result, input_core diff --git a/tools/devsecops_engine_tools/engine_core/src/domain/usecases/handle_scan.py b/tools/devsecops_engine_tools/engine_core/src/domain/usecases/handle_scan.py index acc92dad6..1f388b738 100644 --- a/tools/devsecops_engine_tools/engine_core/src/domain/usecases/handle_scan.py +++ b/tools/devsecops_engine_tools/engine_core/src/domain/usecases/handle_scan.py @@ -19,6 +19,13 @@ from devsecops_engine_tools.engine_core.src.domain.model.vulnerability_management import ( VulnerabilityManagement, ) +from devsecops_engine_tools.engine_core.src.domain.model.gateway.sbom_manager import ( + SbomManagerGateway, +) +from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore +from devsecops_engine_tools.engine_core.src.domain.model.level_vulnerability import ( + LevelVulnerability, +) from devsecops_engine_tools.engine_core.src.domain.model.customs_exceptions import ( ExceptionVulnerabilityManagement, ExceptionFindingsExcepted, @@ -29,10 +36,12 @@ from devsecops_engine_tools.engine_sca.engine_dependencies.src.applications.runner_dependencies_scan import ( runner_engine_dependencies, ) +from devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan import ( + runner_engine_dast +) from devsecops_engine_tools.engine_core.src.infrastructure.helpers.util import ( define_env, ) - from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings @@ -47,68 +56,55 @@ def __init__( vulnerability_management: VulnerabilityManagementGateway, secrets_manager_gateway: SecretsManagerGateway, devops_platform_gateway: DevopsPlatformGateway, + sbom_tool_gateway: SbomManagerGateway, ): self.vulnerability_management = vulnerability_management self.secrets_manager_gateway = secrets_manager_gateway self.devops_platform_gateway = devops_platform_gateway - - def _use_vulnerability_management( - self, config_tool, input_core, dict_args, secret_tool, env - ): - try: - self.vulnerability_management.send_vulnerability_management( - VulnerabilityManagement( - config_tool[dict_args["tool"].upper()]["TOOL"], - input_core, - dict_args, - secret_tool, - config_tool, - self.devops_platform_gateway.get_source_code_management_uri(), - self.devops_platform_gateway.get_base_compact_remote_config_url( - dict_args["remote_config_repo"] - ), - self.devops_platform_gateway.get_variable("access_token"), - self.devops_platform_gateway.get_variable("build_execution_id"), - self.devops_platform_gateway.get_variable("build_id"), - self.devops_platform_gateway.get_variable("branch_tag"), - self.devops_platform_gateway.get_variable("commit_hash"), - env - ) - ) - except ExceptionVulnerabilityManagement as ex1: - logger.error(str(ex1)) - try: - input_core.totalized_exclusions.extend( - self.vulnerability_management.get_findings_excepted( - input_core.scope_pipeline, - dict_args, - secret_tool, - config_tool, - ) - ) - except ExceptionFindingsExcepted as ex2: - logger.error(str(ex2)) + self.sbom_tool_gateway = sbom_tool_gateway def process(self, dict_args: any, config_tool: any): secret_tool = None env = define_env( - self.devops_platform_gateway.get_variable("environment"), - self.devops_platform_gateway.get_variable("branch_name"), - ) + self.devops_platform_gateway.get_variable("environment"), + self.devops_platform_gateway.get_variable("branch_name"), + ) if dict_args["use_secrets_manager"] == "true": secret_tool = self.secrets_manager_gateway.get_secret(config_tool) if "engine_iac" in dict_args["tool"]: findings_list, input_core = runner_engine_iac( - dict_args, config_tool["ENGINE_IAC"]["TOOL"], secret_tool,self.devops_platform_gateway, env + dict_args, + config_tool["ENGINE_IAC"]["TOOL"], + secret_tool, + self.devops_platform_gateway, + env, + ) + self._use_vulnerability_management( + config_tool, input_core, dict_args, secret_tool, env ) - if dict_args["use_vulnerability_management"] == "true" and input_core.path_file_results: - self._use_vulnerability_management( - config_tool, input_core, dict_args, secret_tool, env - ) return findings_list, input_core elif "engine_container" in dict_args["tool"]: - findings_list, input_core = runner_engine_container( - dict_args, config_tool["ENGINE_CONTAINER"]["TOOL"], secret_tool, self.devops_platform_gateway + findings_list, input_core, sbom_components = runner_engine_container( + dict_args, + config_tool["ENGINE_CONTAINER"]["TOOL"], + secret_tool, + self.devops_platform_gateway, + ) + self._use_vulnerability_management( + config_tool, + input_core, + dict_args, + secret_tool, + env, + sbom_components, + ) + return findings_list, input_core + elif "engine_dast" in dict_args["tool"]: + findings_list, input_core = runner_engine_dast( + dict_args, + config_tool["ENGINE_DAST"], + secret_tool, + self.devops_platform_gateway ) if ( dict_args["use_vulnerability_management"] == "true" @@ -118,45 +114,132 @@ def process(self, dict_args: any, config_tool: any): config_tool, input_core, dict_args, secret_tool, env ) return findings_list, input_core - elif "engine_dast" in dict_args["tool"]: - print(MESSAGE_ENABLED) elif "engine_code" in dict_args["tool"]: findings_list, input_core = runner_engine_code( - dict_args, config_tool["ENGINE_CODE"]["TOOL"], self.devops_platform_gateway + dict_args, + config_tool["ENGINE_CODE"]["TOOL"], + self.devops_platform_gateway, + ) + self._use_vulnerability_management( + config_tool, input_core, dict_args, secret_tool, env ) - if ( - dict_args["use_vulnerability_management"] == "true" - and input_core.path_file_results - ): - self._use_vulnerability_management( - config_tool, input_core, dict_args, secret_tool, env - ) return findings_list, input_core elif "engine_secret" in dict_args["tool"]: findings_list, input_core = runner_secret_scan( dict_args, config_tool["ENGINE_SECRET"]["TOOL"], self.devops_platform_gateway, - secret_tool + secret_tool, + ) + self._use_vulnerability_management( + config_tool, input_core, dict_args, secret_tool, env ) - if ( - dict_args["use_vulnerability_management"] == "true" - and input_core.path_file_results - ): - self._use_vulnerability_management( - config_tool, input_core, dict_args, secret_tool, env - ) return findings_list, input_core elif "engine_dependencies" in dict_args["tool"]: - findings_list, input_core = runner_engine_dependencies( - dict_args, config_tool, secret_tool, self.devops_platform_gateway + findings_list, input_core, sbom_components = runner_engine_dependencies( + dict_args, config_tool, secret_tool, self.devops_platform_gateway, self.sbom_tool_gateway + ) + self._use_vulnerability_management( + config_tool, + input_core, + dict_args, + secret_tool, + env, + sbom_components ) + return findings_list, input_core - if ( - dict_args["use_vulnerability_management"] == "true" - and input_core.path_file_results - ): - self._use_vulnerability_management( - config_tool, input_core, dict_args, secret_tool, env + def _define_threshold_quality_vuln( + self, input_core: InputCore, dict_args, secret_tool, config_tool + ): + quality_vulnerability_management = ( + input_core.threshold_defined.quality_vulnerability_management + ) + if quality_vulnerability_management: + product_type = self.vulnerability_management.get_product_type_service( + input_core.scope_pipeline, dict_args, secret_tool, config_tool + ) + if product_type: + pt_name = product_type.name + apply_qualitypt = next( + filter( + lambda qapt: pt_name in qapt, + quality_vulnerability_management["PTS"], + ), + None, + ) + if apply_qualitypt: + pt_info = apply_qualitypt[pt_name] + pt_profile = pt_info["PROFILE"] + pt_apps = pt_info["APPS"] + + input_core.threshold_defined.vulnerability = ( + LevelVulnerability(quality_vulnerability_management[pt_profile]) + if pt_apps == "ALL" + or any(map(lambda pd: pd in input_core.scope_pipeline, pt_apps)) + else input_core.threshold_defined.vulnerability + ) + + def _use_vulnerability_management( + self, + config_tool, + input_core: InputCore, + dict_args, + secret_tool, + env, + sbom_components=None, + ): + if dict_args["use_vulnerability_management"] == "true": + try: + if input_core.path_file_results: + self.vulnerability_management.send_vulnerability_management( + VulnerabilityManagement( + config_tool[dict_args["tool"].upper()]["TOOL"], + input_core, + dict_args, + secret_tool, + config_tool, + self.devops_platform_gateway.get_source_code_management_uri(), + self.devops_platform_gateway.get_base_compact_remote_config_url( + dict_args["remote_config_repo"] + ), + self.devops_platform_gateway.get_variable("access_token"), + self.devops_platform_gateway.get_variable( + "build_execution_id" + ), + self.devops_platform_gateway.get_variable("build_id"), + self.devops_platform_gateway.get_variable("branch_tag"), + self.devops_platform_gateway.get_variable("commit_hash"), + env, + self.devops_platform_gateway.get_variable("vm_product_type_name"), + self.devops_platform_gateway.get_variable("vm_product_name"), + self.devops_platform_gateway.get_variable("vm_product_description"), + ) + ) + + if sbom_components: + self.vulnerability_management.send_sbom_components( + sbom_components, + input_core.scope_pipeline, + dict_args, + secret_tool, + config_tool, + ) + + self._define_threshold_quality_vuln( + input_core, dict_args, secret_tool, config_tool + ) + + except ExceptionVulnerabilityManagement as ex1: + logger.error(str(ex1)) + try: + input_core.totalized_exclusions.extend( + self.vulnerability_management.get_findings_excepted( + input_core.scope_pipeline, + dict_args, + secret_tool, + config_tool, + ) ) - return findings_list, input_core \ No newline at end of file + except ExceptionFindingsExcepted as ex2: + logger.error(str(ex2)) diff --git a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/aws/s3_manager.py b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/aws/s3_manager.py index 72f446269..e29317cbc 100644 --- a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/aws/s3_manager.py +++ b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/aws/s3_manager.py @@ -24,17 +24,22 @@ def _get_s3_data(self, client, bucket, path): return "" def send_metrics(self, config_tool, tool, file_path): - temp_credentials = assume_role( - config_tool["METRICS_MANAGER"]["AWS"]["ROLE_ARN"] - ) + credentials_role = assume_role(config_tool["METRICS_MANAGER"]["AWS"]["ROLE_ARN"]) if config_tool["METRICS_MANAGER"]["AWS"]["USE_ROLE"] else None session = boto3.session.Session() - client = session.client( - service_name="s3", - region_name=config_tool["METRICS_MANAGER"]["AWS"]["REGION_NAME"], - aws_access_key_id=temp_credentials["AccessKeyId"], - aws_secret_access_key=temp_credentials["SecretAccessKey"], - aws_session_token=temp_credentials["SessionToken"], - ) + + if credentials_role: + client = session.client( + service_name="s3", + region_name=config_tool["METRICS_MANAGER"]["AWS"]["REGION_NAME"], + aws_access_key_id=credentials_role["AccessKeyId"], + aws_secret_access_key=credentials_role["SecretAccessKey"], + aws_session_token=credentials_role["SessionToken"], + ) + else: + client = session.client( + service_name="s3", + region_name=config_tool["METRICS_MANAGER"]["AWS"]["REGION_NAME"] + ) date = datetime.datetime.now() path_bucket = f'engine_tools/{tool}/{date.strftime("%Y")}/{date.strftime("%m")}/{date.strftime("%d")}/{file_path.split("/")[-1]}' diff --git a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/aws/secrets_manager.py b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/aws/secrets_manager.py old mode 100644 new mode 100755 index 806e27cfa..f78e8852b --- a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/aws/secrets_manager.py +++ b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/aws/secrets_manager.py @@ -19,15 +19,22 @@ @dataclass class SecretsManager(SecretsManagerGateway): def get_secret(self, config_tool): - temp_credentials = assume_role(config_tool["SECRET_MANAGER"]["AWS"]["ROLE_ARN"]) + credentials_role = assume_role(config_tool["SECRET_MANAGER"]["AWS"]["ROLE_ARN"]) if config_tool["SECRET_MANAGER"]["AWS"]["USE_ROLE"] else None session = boto3.session.Session() - client = session.client( - service_name="secretsmanager", - region_name=config_tool["SECRET_MANAGER"]["AWS"]["REGION_NAME"], - aws_access_key_id=temp_credentials["AccessKeyId"], - aws_secret_access_key=temp_credentials["SecretAccessKey"], - aws_session_token=temp_credentials["SessionToken"], - ) + + if credentials_role: + client = session.client( + service_name="secretsmanager", + region_name=config_tool["SECRET_MANAGER"]["AWS"]["REGION_NAME"], + aws_access_key_id=credentials_role["AccessKeyId"], + aws_secret_access_key=credentials_role["SecretAccessKey"], + aws_session_token=credentials_role["SessionToken"], + ) + else: + client = session.client( + service_name="secretsmanager", + region_name=config_tool["SECRET_MANAGER"]["AWS"]["REGION_NAME"], + ) try: secret_name = config_tool["SECRET_MANAGER"]["AWS"]["SECRET_NAME"] diff --git a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/azure/azure_devops.py b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/azure/azure_devops.py index bf2a52c91..d1f592f50 100644 --- a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/azure/azure_devops.py +++ b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/azure/azure_devops.py @@ -7,6 +7,7 @@ SystemVariables, ReleaseVariables, AgentVariables, + VMVariables ) from devsecops_engine_tools.engine_utilities.azuredevops.infrastructure.azure_devops_api import ( AzureDevopsApi, @@ -23,7 +24,8 @@ @dataclass class AzureDevops(DevopsPlatformGateway): - def get_remote_config(self, repository, path): + def get_remote_config(self, repository, path, branch=""): + base_compact_remote_config_url = ( f"https://{SystemVariables.System_TeamFoundationCollectionUri.value().rstrip('/').split('/')[-1].replace('.visualstudio.com','')}" f".visualstudio.com/{SystemVariables.System_TeamProject.value()}/_git/" @@ -34,7 +36,7 @@ def get_remote_config(self, repository, path): compact_remote_config_url=base_compact_remote_config_url, ) connection = utils_azure.get_azure_connection() - return utils_azure.get_remote_json_config(connection=connection) + return utils_azure.get_remote_json_config(connection=connection, branch=branch) def message(self, type, message): if type == "succeeded": @@ -94,6 +96,9 @@ def get_variable(self, variable): "target_branch": SystemVariables.System_TargetBranchName, "source_branch": SystemVariables.System_SourceBranch, "repository_provider": BuildVariables.Build_Repository_Provider, + "vm_product_type_name": VMVariables.Vm_Product_Type_Name, + "vm_product_name": VMVariables.Vm_Product_Name, + "vm_product_description": VMVariables.Vm_Product_Description, } try: return variable_map.get(variable).value() diff --git a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/defect_dojo/defect_dojo.py b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/defect_dojo/defect_dojo.py index 430861dfe..be477d576 100644 --- a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/defect_dojo/defect_dojo.py +++ b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/defect_dojo/defect_dojo.py @@ -11,6 +11,9 @@ Connect, Finding, Engagement, + Product, + Component, + FindingExclusion ) from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions from devsecops_engine_tools.engine_core.src.domain.model.report import Report @@ -19,7 +22,7 @@ ExceptionVulnerabilityManagement, ExceptionFindingsExcepted, ExceptionGettingFindings, - ExceptionGettingEngagements + ExceptionGettingEngagements, ) from devsecops_engine_tools.engine_core.src.infrastructure.helpers.util import ( format_date, @@ -28,13 +31,24 @@ from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.serializers.import_scan import ( + ImportScanSerializer, +) import time +import concurrent.futures logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() @dataclass class DefectDojoPlatform(VulnerabilityManagementGateway): + + RISK_ACCEPTED = "Risk Accepted" + OUT_OF_SCOPE = "Out of Scope" + FALSE_POSITIVE = "False Positive" + TRANSFERRED_FINDING = "Transferred Finding" + ON_WHITELIST = "On Whitelist" + def send_vulnerability_management( self, vulnerability_management: VulnerabilityManagement ): @@ -66,53 +80,34 @@ def send_vulnerability_management( "KUBESCAPE": "Kubescape Scanner", "KICS": "KICS Scanner", "BEARER": "Bearer CLI", - "DEPENDENCY_CHECK": "Dependency Check Scan" + "DEPENDENCY_CHECK": "Dependency Check Scan", + "SONARQUBE": "SonarQube API Import", + "GITLEAKS": "Gitleaks Scan", + "NUCLEI": "Nuclei Scan" } if any( branch in str(vulnerability_management.branch_tag) for branch in vulnerability_management.config_tool[ "VULNERABILITY_MANAGER" - ]["BRANCH_FILTER"].split(",") + ]["BRANCH_FILTER"] ) or (vulnerability_management.dict_args["tool"] == "engine_secret"): - request: ImportScanRequest = Connect.cmdb( - cmdb_mapping={ - "product_type_name": "nombreevc", - "product_name": "nombreapp", - "tag_product": "nombreentorno", - "product_description": "arearesponsableti", - "codigo_app": "CodigoApp", - }, - compact_remote_config_url=f'{vulnerability_management.base_compact_remote_config_url}{vulnerability_management.config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"]["CMDB_MAPPING_PATH"]}', - personal_access_token=vulnerability_management.access_token, - token_cmdb=token_cmdb, - host_cmdb=vulnerability_management.config_tool[ - "VULNERABILITY_MANAGER" - ]["DEFECT_DOJO"]["HOST_CMDB"], - expression=vulnerability_management.config_tool[ - "VULNERABILITY_MANAGER" - ]["DEFECT_DOJO"]["REGEX_EXPRESSION_CMDB"], - token_defect_dojo=token_dd, - host_defect_dojo=vulnerability_management.config_tool[ - "VULNERABILITY_MANAGER" - ]["DEFECT_DOJO"]["HOST_DEFECT_DOJO"], - scan_type=scan_type_mapping[vulnerability_management.scan_type], - engagement_name=vulnerability_management.input_core.scope_pipeline, - service=vulnerability_management.input_core.scope_pipeline, - file=vulnerability_management.input_core.path_file_results, - version=vulnerability_management.version, - build_id=vulnerability_management.build_id, - source_code_management_uri=vulnerability_management.source_code_management_uri, - branch_tag=vulnerability_management.branch_tag, - commit_hash=vulnerability_management.commit_hash, - environment=( - enviroment_mapping[vulnerability_management.environment.lower()] - if vulnerability_management.environment is not None - and vulnerability_management.environment.lower() - in enviroment_mapping - else enviroment_mapping["default"] - ), - tags=vulnerability_management.dict_args["tool"], + tags = vulnerability_management.dict_args["tool"] + if vulnerability_management.dict_args["tool"] == "engine_iac": + tags = f"{vulnerability_management.dict_args['tool']}_{'_'.join(vulnerability_management.dict_args['platform'])}" + + use_cmdb = vulnerability_management.config_tool[ + "VULNERABILITY_MANAGER" + ]["DEFECT_DOJO"]["CMDB"]["USE_CMDB"] + + request = self._build_request_importscan( + vulnerability_management, + token_cmdb, + token_dd, + scan_type_mapping, + enviroment_mapping, + tags, + use_cmdb, ) def request_func(): @@ -142,6 +137,42 @@ def request_func(): ) ) + def get_product_type_service(self, service, dict_args, secret_tool, config_tool): + try: + session_manager = self._get_session_manager( + dict_args, secret_tool, config_tool + ) + + dd_max_retries = config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"][ + "MAX_RETRIES_QUERY" + ] + + def request_func(): + response = Product.get_product( + session=session_manager, + request={ + "name": Connect.get_code_app( + service, + config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"]["CMDB"][ + "REGEX_EXPRESSION_CMDB" + ], + ), + "prefetch": "prod_type", + }, + ) + return ( + response.prefetch.prod_type[str(response.results[0].prod_type)] + if response.prefetch + else None + ) + + return self._retries_requests(request_func, dd_max_retries, retry_delay=5) + + except Exception as ex: + raise ExceptionVulnerabilityManagement( + "Error getting product type with the following error: {0} ".format(ex) + ) + def get_findings_excepted(self, service, dict_args, secret_tool, config_tool): try: session_manager = self._get_session_manager( @@ -162,6 +193,11 @@ def get_findings_excepted(self, service, dict_args, secret_tool, config_tool): "tags": tool, "limit": dd_limits_query, } + out_of_scope_query_params = { + "out_of_scope": True, + "tags": tool, + "limit": dd_limits_query, + } false_positive_query_params = { "false_p": True, "tags": tool, @@ -172,6 +208,11 @@ def get_findings_excepted(self, service, dict_args, secret_tool, config_tool): "tags": tool, "limit": dd_limits_query, } + white_list_query_params = { + "risk_status": self.ON_WHITELIST, + "tags": tool, + "limit": dd_limits_query, + } exclusions_risk_accepted = self._get_findings_with_exclusions( session_manager, @@ -180,7 +221,7 @@ def get_findings_excepted(self, service, dict_args, secret_tool, config_tool): risk_accepted_query_params, tool, self._format_date_to_dd_format, - "Risk Accepted", + self.RISK_ACCEPTED, ) exclusions_false_positive = self._get_findings_with_exclusions( @@ -190,7 +231,17 @@ def get_findings_excepted(self, service, dict_args, secret_tool, config_tool): false_positive_query_params, tool, self._format_date_to_dd_format, - "False Positive", + self.FALSE_POSITIVE, + ) + + exclusions_out_of_scope = self._get_findings_with_exclusions( + session_manager, + service, + dd_max_retries, + out_of_scope_query_params, + tool, + self._format_date_to_dd_format, + self.OUT_OF_SCOPE, ) exclusions_transfer_finding = self._get_findings_with_exclusions( @@ -200,13 +251,32 @@ def get_findings_excepted(self, service, dict_args, secret_tool, config_tool): transfer_finding_query_params, tool, self._format_date_to_dd_format, - "Transferred Finding", + self.TRANSFERRED_FINDING, + ) + + white_list = self._get_finding_exclusion( + session_manager, dd_max_retries, { + "type": "white_list", + } + ) + + exclusions_white_list = self._get_findings_with_exclusions( + session_manager, + service, + dd_max_retries, + white_list_query_params, + tool, + self._format_date_to_dd_format, + self.ON_WHITELIST, + white_list=white_list, ) return ( list(exclusions_risk_accepted) + list(exclusions_false_positive) + + list(exclusions_out_of_scope) + list(exclusions_transfer_finding) + + list(exclusions_white_list) ) except Exception as ex: raise ExceptionFindingsExcepted( @@ -220,14 +290,20 @@ def get_all(self, service, dict_args, secret_tool, config_tool): all_findings_query_params = { "limit": config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"][ "LIMITS_QUERY" - ] + ], + "duplicate": "false", } max_retries = config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"][ "MAX_RETRIES_QUERY" ] + host_dd = config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"][ + "HOST_DEFECT_DOJO" + ] + + session_manager = self._get_session_manager(dict_args, secret_tool, config_tool) findings = self._get_findings( - self._get_session_manager(dict_args, secret_tool, config_tool), + session_manager, service, max_retries, all_findings_query_params, @@ -235,13 +311,19 @@ def get_all(self, service, dict_args, secret_tool, config_tool): all_findings = list( map( - partial(self._create_report), + partial(self._create_report, host_dd=host_dd), findings, ) ) + white_list = self._get_finding_exclusion( + session_manager, max_retries, { + "type": "white_list", + } + ) + all_exclusions = self._get_report_exclusions( - all_findings, self._format_date_to_dd_format + all_findings, self._format_date_to_dd_format, host_dd=host_dd, white_list=white_list ) return all_findings, all_exclusions @@ -251,7 +333,9 @@ def get_all(self, service, dict_args, secret_tool, config_tool): "Error getting all findings with the following error: {0} ".format(ex) ) - def get_active_engagements(self, engagement_name, dict_args, secret_tool, config_tool): + def get_active_engagements( + self, engagement_name, dict_args, secret_tool, config_tool + ): try: request_is = ImportScanRequest( token_defect_dojo=dict_args.get("token_vulnerability_management") @@ -270,13 +354,140 @@ def get_active_engagements(self, engagement_name, dict_args, secret_tool, config "active": "true", } - return Engagement.get_engagements(request_is, request_active).results + engagements = Engagement.get_engagements(request_is, request_active).results + + host_dd = config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"][ + "HOST_DEFECT_DOJO" + ] + + for engagement in engagements: + engagement.vm_url = f"{host_dd}/engagement/{engagement.id}/finding/open" + + return engagements except Exception as ex: raise ExceptionGettingEngagements( "Error getting engagements with the following error: {0} ".format(ex) ) + def send_sbom_components( + self, sbom_components, service, dict_args, secret_tool, config_tool + ): + try: + engagements = self.get_active_engagements( + service, dict_args, secret_tool, config_tool + ) + engagement = [ + engagement for engagement in engagements if engagement.name == service + ] + session_manager = self._get_session_manager( + dict_args, secret_tool, config_tool + ) + + with concurrent.futures.ThreadPoolExecutor(max_workers=25) as executor: + _ = [ + executor.submit( + self._process_component, + sbom_component, + session_manager, + engagement, + ) + for sbom_component in sbom_components + ] + + except Exception as ex: + raise ExceptionVulnerabilityManagement( + "Error sending components sbom to vulnerability management with the following error: {0} ".format( + ex + ) + ) + + def _build_request_importscan( + self, + vulnerability_management: VulnerabilityManagement, + token_cmdb, + token_dd, + scan_type_mapping, + enviroment_mapping, + tags, + use_cmdb: bool, + ): + common_fields = { + "scan_type": scan_type_mapping[vulnerability_management.scan_type], + "file": vulnerability_management.input_core.path_file_results, + "engagement_name": vulnerability_management.input_core.scope_pipeline, + "source_code_management_uri": vulnerability_management.source_code_management_uri, + "tags": tags, + "version": vulnerability_management.version, + "build_id": vulnerability_management.build_id, + "branch_tag": vulnerability_management.branch_tag, + "commit_hash": vulnerability_management.commit_hash, + "service": vulnerability_management.input_core.scope_pipeline, + "environment": ( + enviroment_mapping[vulnerability_management.environment.lower()] + if vulnerability_management.environment is not None + and vulnerability_management.environment.lower() in enviroment_mapping + else enviroment_mapping["default"] + ), + "token_defect_dojo": token_dd, + "host_defect_dojo": vulnerability_management.config_tool[ + "VULNERABILITY_MANAGER" + ]["DEFECT_DOJO"]["HOST_DEFECT_DOJO"], + "expression": vulnerability_management.config_tool["VULNERABILITY_MANAGER"][ + "DEFECT_DOJO" + ]["CMDB"]["REGEX_EXPRESSION_CMDB"], + } + + if use_cmdb: + cmdb_mapping = vulnerability_management.config_tool[ + "VULNERABILITY_MANAGER" + ]["DEFECT_DOJO"]["CMDB"]["CMDB_MAPPING"] + return Connect.cmdb( + cmdb_mapping={ + "product_type_name": cmdb_mapping["PRODUCT_TYPE_NAME"], + "product_name": cmdb_mapping["PRODUCT_NAME"], + "tag_product": cmdb_mapping["TAG_PRODUCT"], + "product_description": cmdb_mapping["PRODUCT_DESCRIPTION"], + "codigo_app": cmdb_mapping["CODIGO_APP"], + }, + compact_remote_config_url=f'{vulnerability_management.base_compact_remote_config_url}{vulnerability_management.config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"]["CMDB"]["CMDB_MAPPING_PATH"]}', + personal_access_token=vulnerability_management.access_token, + token_cmdb=token_cmdb, + host_cmdb=vulnerability_management.config_tool["VULNERABILITY_MANAGER"][ + "DEFECT_DOJO" + ]["CMDB"]["HOST_CMDB"], + cmdb_request_response=vulnerability_management.config_tool[ + "VULNERABILITY_MANAGER" + ]["DEFECT_DOJO"]["CMDB"]["CMDB_REQUEST_RESPONSE"], + **common_fields, + ) + else: + request: ImportScanRequest = ImportScanSerializer().load( + { + "product_type_name": vulnerability_management.vm_product_type_name, + "product_name": vulnerability_management.vm_product_name, + "product_description": vulnerability_management.vm_product_description, + "code_app": vulnerability_management.vm_product_name, + **common_fields, + } + ) + return request + + def _process_component(self, component_sbom, session_manager, engagement): + request = { + "name": component_sbom.name, + "version": component_sbom.version, + "engagement_id": engagement[0].id, + } + components = Component.get_component(session=session_manager, request=request) + if components.results == []: + response = Component.create_component( + session=session_manager, request=request + ) + logger.info( + f"Component created: {response.name} - {response.version} found with id: {response.id}" + ) + def _get_session_manager(self, dict_args, secret_tool, config_tool): token_dd = dict_args.get("token_vulnerability_management") or secret_tool.get( "token_defect_dojo" @@ -286,37 +497,55 @@ def _get_session_manager(self, dict_args, secret_tool, config_tool): config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"]["HOST_DEFECT_DOJO"], ) - def _get_report_exclusions(self, total_findings, date_fn): + def _get_report_exclusions(self, total_findings, date_fn, host_dd, **kwargs): exclusions = [] for finding in total_findings: if finding.risk_accepted: exclusions.append( - self._create_exclusion( - finding, date_fn, "engine_risk", "Risk Accepted" + self._create_report_exclusion( + finding, date_fn, "engine_risk", self.RISK_ACCEPTED, host_dd, **kwargs ) ) elif finding.false_p: exclusions.append( - self._create_exclusion( - finding, date_fn, "engine_risk", "False Positive" + self._create_report_exclusion( + finding, date_fn, "engine_risk", self.FALSE_POSITIVE, host_dd, **kwargs + ) + ) + elif finding.out_of_scope: + exclusions.append( + self._create_report_exclusion( + finding, date_fn, "engine_risk", self.OUT_OF_SCOPE, host_dd, **kwargs ) ) elif finding.risk_status == "Transfer Accepted": exclusions.append( - self._create_exclusion( - finding, date_fn, "engine_risk", "Transferred Finding" + self._create_report_exclusion( + finding, + date_fn, + "engine_risk", + self.TRANSFERRED_FINDING, + host_dd, + **kwargs + ) + ) + elif finding.risk_status == self.ON_WHITELIST: + exclusions.append( + self._create_report_exclusion( + finding, date_fn, "engine_risk", self.ON_WHITELIST, host_dd, **kwargs ) ) return exclusions def _get_findings_with_exclusions( - self, session_manager, service, max_retries, query_params, tool, date_fn, reason + self, session_manager, service, max_retries, query_params, tool, date_fn, reason, **kwargs ): findings = self._get_findings( session_manager, service, max_retries, query_params ) + return map( - partial(self._create_exclusion, date_fn=date_fn, tool=tool, reason=reason), + partial(self._create_exclusion, date_fn=date_fn, tool=tool, reason=reason, **kwargs), findings, ) @@ -327,6 +556,14 @@ def request_func(): ).results return self._retries_requests(request_func, max_retries, retry_delay=5) + + def _get_finding_exclusion(self, session_manager, max_retries, query_params): + def request_func(): + return FindingExclusion.get_finding_exclusion( + session=session_manager, **query_params + ).results + + return self._retries_requests(request_func, max_retries, retry_delay=5) def _retries_requests(self, request_func, max_retries, retry_delay): for attempt in range(max_retries): @@ -341,20 +578,44 @@ def _retries_requests(self, request_func, max_retries, retry_delay): logger.error("Maximum number of retries reached, aborting.") raise e - def _create_exclusion(self, finding, date_fn, tool, reason): - if reason == "False Positive": - create_date = date_fn(finding.last_status_update) - expired_date = date_fn(None) - elif reason == "Transferred Finding": - create_date = date_fn(finding.transfer_finding.date) - expired_date = date_fn(finding.transfer_finding.expiration_date) - else: - last_accepted_risk = finding.accepted_risks[-1] - create_date = date_fn(last_accepted_risk["created"]) - expired_date = date_fn(last_accepted_risk["expiration_date"]) + def _date_reason_based(self, finding, date_fn, reason, tool, **kwargs): + def get_vuln_id(finding, tool): + if tool == "engine_risk": + return finding.id[0]["vulnerability_id"] if finding.id else finding.vuln_id_from_tool + else: + return finding.vulnerability_ids[0]["vulnerability_id"] if finding.vulnerability_ids else finding.vuln_id_from_tool + + def get_dates_from_whitelist(vuln_id, white_list): + matching_finding = next(filter(lambda x: x.unique_id_from_tool == vuln_id, white_list), None) + if matching_finding: + return date_fn(matching_finding.create_date), date_fn(matching_finding.expiration_date) + return date_fn(None), date_fn(None) + + reason_to_dates = { + self.FALSE_POSITIVE: lambda: (date_fn(finding.last_status_update), date_fn(None)), + self.OUT_OF_SCOPE: lambda: (date_fn(finding.last_status_update), date_fn(None)), + self.TRANSFERRED_FINDING: lambda: (date_fn(finding.transfer_finding.date), date_fn(finding.transfer_finding.expiration_date)), + self.RISK_ACCEPTED: lambda: (date_fn(finding.accepted_risks[-1]["created"]), date_fn(finding.accepted_risks[-1]["expiration_date"])), + self.ON_WHITELIST: lambda: get_dates_from_whitelist(get_vuln_id(finding, tool), kwargs.get("white_list", [])), + } + + create_date, expired_date = reason_to_dates.get(reason, lambda: (date_fn(None), date_fn(None)))() + return create_date, expired_date + + def _create_exclusion(self, finding, date_fn, tool, reason, **kwargs): + create_date, expired_date = self._date_reason_based(finding, date_fn, reason, tool, **kwargs) + return Exclusions( - id=finding.vuln_id_from_tool, + id=( + finding.vuln_id_from_tool + if finding.vuln_id_from_tool + else ( + finding.vulnerability_ids[0]["vulnerability_id"] + if finding.vulnerability_ids + else "" + ) + ), where=self._get_where(finding, tool), create_date=create_date, expired_date=expired_date, @@ -362,8 +623,30 @@ def _create_exclusion(self, finding, date_fn, tool, reason): reason=reason, ) - def _create_report(self, finding): + def _create_report_exclusion(self, finding, date_fn, tool, reason, host_dd, **kwargs): + create_date, expired_date = self._date_reason_based(finding, date_fn, reason, tool, **kwargs) + + return Exclusions( + id=( + finding.vuln_id_from_tool + if finding.vuln_id_from_tool + else finding.id[0]["vulnerability_id"] if finding.id else "" + ), + where=self._get_where(finding, tool), + create_date=create_date, + expired_date=expired_date, + severity=finding.severity, + reason=reason, + vm_id=str(finding.vm_id), + vm_id_url=f"{host_dd}/finding/{finding.vm_id}", + service=finding.service, + tags=finding.tags, + ) + + def _create_report(self, finding, host_dd): return Report( + vm_id=str(finding.id), + vm_id_url=f"{host_dd}/finding/{finding.id}", id=finding.vulnerability_ids, vuln_id_from_tool=finding.vuln_id_from_tool, status=finding.display_status, @@ -389,7 +672,9 @@ def _create_report(self, finding): vul_description=finding.description, risk_accepted=finding.risk_accepted, false_p=finding.false_p, + out_of_scope=finding.out_of_scope, service=finding.service, + unique_id_from_tool=finding.unique_id_from_tool, ) def _format_date_to_dd_format(self, date_string): @@ -400,12 +685,19 @@ def _format_date_to_dd_format(self, date_string): ) def _get_where(self, finding, tool): - if tool in ["engine_container", "engine_dependencies"]: + if tool == "engine_dependencies": + return ( + finding.component_name.replace("_", ":") + + ":" + + finding.component_version + ) + elif tool == "engine_container": return finding.component_name + ":" + finding.component_version elif tool == "engine_dast": return finding.endpoints elif tool == "engine_risk": for tag in finding.tags: return self._get_where(finding, tag) + return finding.file_path else: return finding.file_path diff --git a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/github/github_actions.py b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/github/github_actions.py old mode 100644 new mode 100755 index 23948f5e8..9ccd76791 --- a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/github/github_actions.py +++ b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/github/github_actions.py @@ -6,7 +6,8 @@ BuildVariables, SystemVariables, ReleaseVariables, - AgentVariables + AgentVariables, + VMVariables ) from devsecops_engine_tools.engine_utilities.github.infrastructure.github_api import ( GithubApi, @@ -22,18 +23,15 @@ class GithubActions(DevopsPlatformGateway): ICON_FAIL = "\u2718" ICON_SUCCESS = "\u2714" - def get_remote_config(self, repository, path): + def get_remote_config(self, repository, path, branch=""): github_repository = SystemVariables.github_repository.value() split = github_repository.split("/") owner = split[0] - utils_github = GithubApi( - personal_access_token=SystemVariables.github_access_token.value() - ) - - git_client = utils_github.get_github_connection() - json_config = utils_github.get_remote_json_config(git_client, owner, repository, path) + utils_github = GithubApi() + git_client = utils_github.get_github_connection(SystemVariables.github_access_token.value()) + json_config = utils_github.get_remote_json_config(git_client, owner, repository, path, branch) return json_config @@ -88,8 +86,11 @@ def get_variable(self, variable): "target_branch": SystemVariables.github_event_base_ref, "source_branch": SystemVariables.github_ref, "repository_provider": BuildVariables.GitHub, + "vm_product_type_name": VMVariables.Vm_Product_Type_Name, + "vm_product_name": VMVariables.Vm_Product_Name, + "vm_product_description": VMVariables.Vm_Product_Description, } try: return variable_map.get(variable).value() except ValueError: - return None + return None \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/printer_pretty_table/printer_pretty_table.py b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/printer_pretty_table/printer_pretty_table.py index 20cf7072c..0d5ef512b 100644 --- a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/printer_pretty_table/printer_pretty_table.py +++ b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/printer_pretty_table/printer_pretty_table.py @@ -10,7 +10,7 @@ Report, ) from devsecops_engine_tools.engine_core.src.infrastructure.helpers.util import ( - format_date + format_date, ) from prettytable import PrettyTable, DOUBLE_BORDER @@ -63,24 +63,20 @@ def print_table_findings(self, finding_list: "list[Finding]"): print(sorted_table) def print_table_report(self, report_list: "list[Report]"): - headers = ["Risk Score", "Severity", "ID", "Tags", "Where", "Service"] + headers = ["Risk Score", "VM ID", "Services", "Tags"] table = PrettyTable(headers) for report in report_list: row_data = [ report.risk_score, - report.severity.lower(), - report.vuln_id_from_tool if report.vuln_id_from_tool else report.id, - report.tags, - report.where, - report.service + self._check_spaces(report.vm_id), + self._check_spaces(report.service), + ", ".join(report.tags), ] table.add_row(row_data) sorted_table = PrettyTable() sorted_table.field_names = table.field_names - sorted_table.add_rows( - sorted(table._rows, key=lambda row: row[0], reverse=True) - ) + sorted_table.add_rows(sorted(table._rows, key=lambda row: row[0], reverse=True)) for column in table.field_names: sorted_table.align[column] = "l" @@ -90,9 +86,52 @@ def print_table_report(self, report_list: "list[Report]"): if len(sorted_table.rows) > 0: print(sorted_table) + def print_table_report_exlusions(self, exclusions): + if exclusions: + headers = [ + "VM ID", + "Services", + "Tags", + "Created Date", + "Expired Date", + "Reason", + ] + + table = PrettyTable(headers) + + for exclusion in exclusions: + row_data = [ + self._check_spaces(exclusion["vm_id"]), + self._check_spaces(exclusion["service"]), + ", ".join(exclusion["tags"]), + format_date(exclusion["create_date"], "%d%m%Y", "%d/%m/%Y"), + ( + format_date(exclusion["expired_date"], "%d%m%Y", "%d/%m/%Y") + if exclusion["expired_date"] + and exclusion["expired_date"] != "undefined" + else "NA" + ), + exclusion["reason"], + ] + table.add_row(row_data) + + for column in table.field_names: + table.align[column] = "l" + + table.set_style(DOUBLE_BORDER) + if len(table.rows) > 0: + print(table) + def print_table_exclusions(self, exclusions): - if (exclusions): - headers = ["Severity", "ID", "Where", "Create Date", "Expired Date", "Reason"] + if exclusions: + headers = [ + "Severity", + "ID", + "Where", + "Create Date", + "Expired Date", + "Reason", + ] table = PrettyTable(headers) @@ -102,7 +141,12 @@ def print_table_exclusions(self, exclusions): exclusion["id"], exclusion["where"], format_date(exclusion["create_date"], "%d%m%Y", "%d/%m/%Y"), - format_date(exclusion["expired_date"], "%d%m%Y", "%d/%m/%Y") if exclusion["expired_date"] and exclusion["expired_date"] != "undefined" else "NA", + ( + format_date(exclusion["expired_date"], "%d%m%Y", "%d/%m/%Y") + if exclusion["expired_date"] + and exclusion["expired_date"] != "undefined" + else "NA" + ), exclusion["reason"], ] table.add_row(row_data) @@ -113,3 +157,12 @@ def print_table_exclusions(self, exclusions): table.set_style(DOUBLE_BORDER) if len(table.rows) > 0: print(table) + + def _check_spaces(self, value): + values = value.split() + new_value = "" + if len(values) > 1: + new_value = "\n".join(values) + else: + new_value = f"{values[0]}" + return new_value diff --git a/tools/devsecops_engine_tools/engine_core/test/.gitkeep b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/printer_rich_table/__init__.py similarity index 100% rename from tools/devsecops_engine_tools/engine_core/test/.gitkeep rename to tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/printer_rich_table/__init__.py diff --git a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/printer_rich_table/printer_rich_table.py b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/printer_rich_table/printer_rich_table.py new file mode 100644 index 000000000..ac9a499bc --- /dev/null +++ b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/printer_rich_table/printer_rich_table.py @@ -0,0 +1,86 @@ +from dataclasses import dataclass + +from devsecops_engine_tools.engine_core.src.domain.model.gateway.printer_table_gateway import ( + PrinterTableGateway, +) +from devsecops_engine_tools.engine_core.src.domain.model.finding import ( + Finding, +) +from devsecops_engine_tools.engine_core.src.domain.model.report import ( + Report, +) +from devsecops_engine_tools.engine_core.src.infrastructure.helpers.util import ( + format_date, +) +from rich.console import Console +from rich.table import Table +from rich import box + + +@dataclass +class PrinterRichTable(PrinterTableGateway): + def print_table_findings(self, finding_list: "list[Finding]"): + # To implement + return + + def print_table_report(self, report_list: "list[Report]"): + sorted_report_list = sorted( + report_list, key=lambda report: report.risk_score, reverse=True + ) + headers = ["Risk Score", "ID", "Tags", "Services"] + table = Table( + show_header=True, header_style="bold magenta", box=box.DOUBLE_EDGE + ) + for header in headers: + table.add_column(header) + for report in sorted_report_list: + row_data = [ + str(report.risk_score), + self._check_spaces(report.vm_id, report.vm_id_url), + ", ".join(report.tags), + report.service, + ] + table.add_row(*row_data) + console = Console() + console.print(table) + + def print_table_exclusions(self, exclusions_list): + headers = [] + if exclusions_list: + headers = ["ID", "Tags", "Service", "Create Date", "Expired Date", "Reason"] + table = Table( + show_header=True, header_style="bold magenta", box=box.DOUBLE_EDGE + ) + for header in headers: + table.add_column(header) + for exclusion in exclusions_list: + row_data = [ + self._check_spaces(exclusion["vm_id"], exclusion["vm_id_url"]), + ", ".join(exclusion["tags"]), + exclusion["service"], + format_date(exclusion["create_date"], "%d%m%Y", "%d/%m/%Y"), + ( + format_date(exclusion["expired_date"], "%d%m%Y", "%d/%m/%Y") + if exclusion["expired_date"] + and exclusion["expired_date"] != "undefined" + else "NA" + ), + exclusion["reason"], + ] + table.add_row(*row_data) + console = Console() + console.print(table) + + def _check_spaces(self, value, url): + values = value.split() + urls = url.split() + new_value = "" + if len(values) > 1 or len(urls) > 1: + for value, url in zip(values, urls): + new_value += self._make_hyperlink(value, url) + " " + else: + new_value = self._make_hyperlink(values[0], urls[0]) + return new_value + + def _make_hyperlink(self, value, url): + return f"[link={url}]{value}[/link]" diff --git a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/runtime_local/runtime_local.py b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/runtime_local/runtime_local.py old mode 100644 new mode 100755 index 170af803b..e27168201 --- a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/runtime_local/runtime_local.py +++ b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/runtime_local/runtime_local.py @@ -18,8 +18,10 @@ class RuntimeLocal(DevopsPlatformGateway): ICON_SUCCESS = "\u2714" - def get_remote_config(self, repository, path): - with open(f"{repository}/{path}") as f: + def get_remote_config(self, repository, path, branch=""): + remote_config_path = f"{repository}/{path}" + + with open(remote_config_path, 'r', encoding='utf-8') as f: return json.load(f) def message(self, type, message): @@ -42,7 +44,7 @@ def get_source_code_management_uri(self): return os.environ.get("DET_SOURCE_CODE_MANAGEMENT_URI") def get_base_compact_remote_config_url(self, remote_config_repo): - return os.environ.get("DET_BASE_COMPACT_REMOTE_CONFIG_URL") + return f"{os.environ.get('DET_BASE_COMPACT_REMOTE_CONFIG_URL')}?path=/" def get_variable(self, variable): env_variables = { @@ -64,6 +66,9 @@ def get_variable(self, variable): "temp_directory" : "DET_TEMP_DIRECTORY", "target_branch" : "DET_TARGET_BRANCH", "source_branch" : "DET_SOURCE_BRANCH", - "repository_provider" : "DET_REPOSITORY_PROVIDER" + "repository_provider" : "DET_REPOSITORY_PROVIDER", + "vm_product_type_name" : "DET_VM_PRODUCT_TYPE_NAME", + "vm_product_name" : "DET_VM_PRODUCT_NAME", + "vm_product_description" : "DET_VM_PRODUCT_DESCRIPTION", } return os.environ.get(env_variables[variable], None) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/applications/.gitkeep b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/syft/__init__.py similarity index 100% rename from tools/devsecops_engine_tools/engine_dast/src/applications/.gitkeep rename to tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/syft/__init__.py diff --git a/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/syft/syft.py b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/syft/syft.py new file mode 100644 index 000000000..7d2a7ecd5 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/syft/syft.py @@ -0,0 +1,122 @@ +from dataclasses import dataclass +import requests +import subprocess +import tarfile +import zipfile +import platform + +from devsecops_engine_tools.engine_core.src.domain.model.gateway.sbom_manager import ( + SbomManagerGateway, +) +from devsecops_engine_tools.engine_utilities.sbom.deserealizator import ( + get_list_component, +) +from devsecops_engine_tools.engine_core.src.domain.model.component import ( + Component, +) + +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities import settings + +logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() + + +@dataclass +class Syft(SbomManagerGateway): + + def get_components(self, artifact, config, service_name) -> "list[Component]": + try: + syft_version = config["SYFT"]["SYFT_VERSION"] + os_platform = platform.system() + base_url = ( + f"https://github.com/anchore/syft/releases/download/v{syft_version}/" + ) + + command_prefix = "syft" + if os_platform == "Linux": + file = f"syft_{syft_version}_linux_amd64.tar.gz" + command_prefix = self._install_tool_unix( + file, base_url + file, command_prefix + ) + elif os_platform == "Darwin": + file = f"syft_{syft_version}_darwin_amd64.tar.gz" + command_prefix = self._install_tool_unix( + file, base_url + file, command_prefix + ) + elif os_platform == "Windows": + file = f"syft_{syft_version}_windows_amd64.zip" + command_prefix = self._install_tool_windows( + file, base_url + file, "syft.exe" + ) + else: + logger.warning(f"{os_platform} is not supported.") + return None + + result_sbom = self._run_syft(command_prefix, artifact, config, service_name) + return get_list_component(result_sbom, config["SYFT"]["OUTPUT_FORMAT"]) + except Exception as e: + logger.error(f"Error generating SBOM: {e}") + return None + + def _run_syft(self, command_prefix, artifact, config, service_name): + result_file = f"{service_name}_SBOM.json" + command = [ + command_prefix, + artifact, + "-o", + f"{config['SYFT']['OUTPUT_FORMAT']}={result_file}", + ] + try: + subprocess.run( + command, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + print(f"SBOM generated and saved to: {result_file}") + return result_file + except Exception as e: + logger.error(f"Error running syft: {e}") + + def _install_tool_unix(self, file, url, command_prefix): + installed = subprocess.run( + ["which", command_prefix], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if installed.returncode == 1: + try: + self._download_tool(file, url) + with tarfile.open(file, "r:gz") as tar_file: + tar_file.extract(member=tar_file.getmember("syft")) + return "./syft" + except Exception as e: + logger.error(f"Error installing syft: {e}") + else: + return installed.stdout.decode("utf-8").strip() + + def _install_tool_windows(self, file, url, command_prefix): + try: + installed = subprocess.run( + [command_prefix, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return installed.stdout.decode("utf-8").strip() + except: + try: + self._download_tool(file, url) + with zipfile.ZipFile(file, "r") as zip_file: + zip_file.extract(member="syft.exe") + return "./syft.exe" + except Exception as e: + logger.error(f"Error installing syft: {e}") + + def _download_tool(self, file, url): + try: + response = requests.get(url, allow_redirects=True) + with open(file, "wb") as compress_file: + compress_file.write(response.content) + except Exception as e: + logger.error(f"Error downloading syft: {e}") diff --git a/tools/devsecops_engine_tools/engine_core/src/infrastructure/entry_points/entry_point_core.py b/tools/devsecops_engine_tools/engine_core/src/infrastructure/entry_points/entry_point_core.py index 51ed25d6b..04fc1e6da 100644 --- a/tools/devsecops_engine_tools/engine_core/src/infrastructure/entry_points/entry_point_core.py +++ b/tools/devsecops_engine_tools/engine_core/src/infrastructure/entry_points/entry_point_core.py @@ -21,14 +21,15 @@ def init_engine_core( devops_platform_gateway: any, print_table_gateway: any, metrics_manager_gateway: any, + sbom_tool_gateway: any, args: any ): config_tool = devops_platform_gateway.get_remote_config( - args["remote_config_repo"], "/engine_core/ConfigTool.json" + args["remote_config_repo"], "/engine_core/ConfigTool.json", args["remote_config_branch"] ) Printers.print_logo_tool(config_tool["BANNER"]) - if config_tool[args["tool"].upper()]["ENABLED"] == "true": + if config_tool[args["tool"].upper()]["ENABLED"]: if args["tool"] == "engine_risk": results, input_core = HandleRisk( vulnerability_management_gateway, @@ -42,6 +43,7 @@ def init_engine_core( vulnerability_management_gateway, secrets_manager_gateway, devops_platform_gateway, + sbom_tool_gateway ).process(args, config_tool) results = BreakBuild(devops_platform_gateway, print_table_gateway).process( diff --git a/tools/devsecops_engine_tools/engine_core/test/applications/test_runner_engine_core.py b/tools/devsecops_engine_tools/engine_core/test/applications/test_runner_engine_core.py index e986d64cf..28a38e514 100644 --- a/tools/devsecops_engine_tools/engine_core/test/applications/test_runner_engine_core.py +++ b/tools/devsecops_engine_tools/engine_core/test/applications/test_runner_engine_core.py @@ -17,6 +17,7 @@ def test_application_core(mock_get_inputs_from_cli, mock_entry_point_tool): mock_args = { "platform_devops": "azure", "remote_config_repo": "https://github.com/example/repo", + "remote_config_branch": "", "tool": "engine_iac", "environment": "dev", "platform": "k8s", @@ -28,6 +29,7 @@ def test_application_core(mock_get_inputs_from_cli, mock_entry_point_tool): "token_engine_container": None, "token_engine_dependencies": None, "xray_mode": "scan", + "dast_file_path": "dast_file_path", } # Mock the dependencies @@ -55,6 +57,7 @@ def test_application_core_exception( mock_args = { "platform_devops": "azure", "remote_config_repo": "https://github.com/example/repo", + "remote_config_branch": "", "tool": "engine_iac", "environment": "dev", "platform": "all", @@ -87,6 +90,7 @@ def test_get_inputs_from_cli(mock_parse_args): mock_args = mock.MagicMock() mock_args.platform_devops = "azure" mock_args.remote_config_repo = "https://github.com/example/repo" + mock_args.remote_config_branch = "" mock_args.tool = "engine_iac" mock_args.folder_path = "/path/to/folder" mock_args.platform = "k8s,docker" @@ -100,6 +104,7 @@ def test_get_inputs_from_cli(mock_parse_args): mock_args.token_external_checks = None mock_args.xray_mode = "scan" mock_args.image_to_scan = "image" + mock_args.dast_file_path = "dast_file_path" # Mock the parse_args method mock_parse_args.return_value = mock_args @@ -111,6 +116,7 @@ def test_get_inputs_from_cli(mock_parse_args): assert result == { "platform_devops": "azure", "remote_config_repo": "https://github.com/example/repo", + "remote_config_branch": "", "tool": "engine_iac", "folder_path": "/path/to/folder", "platform": "k8s,docker", @@ -124,6 +130,7 @@ def test_get_inputs_from_cli(mock_parse_args): "token_external_checks": None, "xray_mode": "scan", "image_to_scan":"image", + "dast_file_path": "dast_file_path" } @@ -132,4 +139,4 @@ def test_parse_choices(): result = parse_separated_list( "docker,k8s", {"all", "docker", "k8s", "cloudformation"} ) - assert result == ["docker", "k8s"] + assert result == ["docker", "k8s"] \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_break_build.py b/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_break_build.py index ad934593b..29b42f5b2 100644 --- a/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_break_build.py +++ b/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_break_build.py @@ -32,15 +32,6 @@ def test_process_no_findings(self, mock_print): "Medium": 10, "Low": 15, }, - "CUSTOM_VULNERABILITY": { - "PATTERN_APPS": "^(?!App1$).*(App2.*|.*App3.*)", - "VULNERABILITY": { - "Critical": 0, - "High": 0, - "Medium": 5, - "Low": 10, - }, - }, "COMPLIANCE": {"Critical": 1}, "CVE": ["CKV_K8S_22"], } diff --git a/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_handle_risk.py b/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_handle_risk.py index 66edc9e6b..eb33fcff7 100644 --- a/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_handle_risk.py +++ b/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_handle_risk.py @@ -23,6 +23,9 @@ def setUp(self): self.print_table_gateway, ) + @mock.patch( + "devsecops_engine_tools.engine_core.src.domain.usecases.handle_risk.HandleRisk._should_skip_analysis" + ) @mock.patch( "devsecops_engine_tools.engine_core.src.domain.usecases.handle_risk.runner_engine_risk" ) @@ -39,11 +42,13 @@ def test_process( mock_filter_engagements, mock_get_all_from_vm, mock_runner_engine_risk, + mock_should_skip_analysis, ): dict_args = { "use_secrets_manager": "true", "tool": "engine_risk", "remote_config_repo": "test_repo", + "remote_config_branch": "" } config_tool = {"ENGINE_RISK": {"ENABLED": "true"}} self.devops_platform_gateway.get_remote_config.return_value = { @@ -51,16 +56,20 @@ def test_process( "HANDLE_SERVICE_NAME": { "ENABLED": "true", "ADD_SERVICES": ["service1", "service2"], - "ERASE_SERVICE_ENDING": ["_ending"], + "CHECK_ENDING": ["_ending"], "REGEX_GET_SERVICE_CODE": "[^_]+", }, } self.devops_platform_gateway.get_variable.return_value = ( "code_pipeline_name_id_test" ) + mock_should_skip_analysis.return_value = False mock_runner_engine_risk.return_value = {"result": "result"} mock_get_all_from_vm.return_value = ([], []) - mock_filter_engagements.return_value = ["service1", "service2"] + mock_filter_engagements.return_value = [ + MagicMock(name="service1"), + MagicMock(name="service2"), + ] mock_match.side_effect = [ MagicMock(group=MagicMock(return_value="code_pipeline_name_id_test")), MagicMock(group=MagicMock(return_value="code_pipeline_name_id_test")), @@ -72,7 +81,7 @@ def test_process( # Assert the expected values assert mock_filter_engagements.call_count == 1 assert mock_match.call_count == 2 - assert mock_get_all_from_vm.call_count == 3 + assert mock_get_all_from_vm.call_count == 2 assert mock_runner_engine_risk.call_count == 1 assert result == {"result": "result"} assert type(input_core) == InputCore @@ -88,8 +97,15 @@ def test_filter_engagements(self, mock_search): MagicMock(name="code_another_service_2"), ] service = "code_service_id" + initial_services = [ + "code_service_id", + "code_service_id_test", + "code_service_id_test_word1", + "code_service_id_test_word2", + ] risk_config = { "HANDLE_SERVICE_NAME": { + "CHECK_ENDING": ["_ending"], "REGEX_GET_WORDS": "[_-]", "MIN_WORD_LENGTH": 3, "MIN_WORD_AMOUNT": 2, @@ -98,7 +114,9 @@ def test_filter_engagements(self, mock_search): } # Call the process method - self.handle_risk._filter_engagements(engagements, service, risk_config) + self.handle_risk._filter_engagements( + engagements, service, initial_services, risk_config + ) # Assert the expected values mock_search.assert_called() @@ -108,6 +126,7 @@ def test_get_all_from_vm(self): "use_secrets_manager": "true", "tool": "engine_risk", "remote_config_repo": "test_repo", + "remote_config_branch": "" } secret_tool = None remote_config = {"ENGINE_RISK": {"ENABLED": "true"}} @@ -129,6 +148,7 @@ def test_get_all_from_vm_exception(self, mock_logger_error): "use_secrets_manager": "true", "tool": "engine_risk", "remote_config_repo": "test_repo", + "remote_config_branch": "" } secret_tool = None remote_config = {"ENGINE_RISK": {"ENABLED": "true"}} @@ -150,9 +170,15 @@ def test_get_all_from_vm_exception(self, mock_logger_error): def test_exclude_services(self): dict_args = { "remote_config_repo": "test_repo", + "remote_config_branch": "" } pipeline_name = "pipeline_name" - service_list = ["code_service_1", "code_service_2", "service1", "service2"] + service_list = [ + MagicMock(name="code_service_1"), + MagicMock(name="code_service_2"), + MagicMock(name="service_1"), + MagicMock(name="service_2"), + ] self.devops_platform_gateway.get_remote_config.return_value = { "pipeline_name": { "SKIP_SERVICE": {"services": ["code_service_1", "code_service_2"]} @@ -166,3 +192,14 @@ def test_exclude_services(self): # Assert the expected values assert type(result) == list + + def test_should_skip_analysis(self): + remote_config = {"IGNORE_ANALYSIS_PATTERN": "pattern"} + pipeline_name = "pipeline" + exclusions = {"pipeline": {"SKIP_TOOL": 1}} + + result = self.handle_risk._should_skip_analysis( + remote_config, pipeline_name, exclusions + ) + + assert result == True diff --git a/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_handle_scan.py b/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_handle_scan.py index f74dd7543..0962382c2 100644 --- a/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_handle_scan.py +++ b/tools/devsecops_engine_tools/engine_core/test/domain/usecases/test_handle_scan.py @@ -1,12 +1,16 @@ import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock from unittest import mock from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold +from devsecops_engine_tools.engine_core.src.domain.model.component import Component from devsecops_engine_tools.engine_core.src.domain.usecases.handle_scan import ( HandleScan, ) -from devsecops_engine_tools.engine_core.src.domain.model.customs_exceptions import ( ExceptionVulnerabilityManagement, ExceptionFindingsExcepted) +from devsecops_engine_tools.engine_core.src.domain.model.customs_exceptions import ( + ExceptionVulnerabilityManagement, + ExceptionFindingsExcepted, +) class TestHandleScan(unittest.TestCase): @@ -14,10 +18,23 @@ def setUp(self): self.vulnerability_management = MagicMock() self.secrets_manager_gateway = MagicMock() self.devops_platform_gateway = MagicMock() + self.sbom_gateway = MagicMock() + self.threshold = Threshold( + { + "VULNERABILITY": { + "Critical": 5, + "High": 8, + "Medium": 10, + "Low": 15, + }, + "COMPLIANCE": {"Critical": 1}, + } + ) self.handle_scan = HandleScan( self.vulnerability_management, self.secrets_manager_gateway, self.devops_platform_gateway, + self.sbom_gateway, ) @mock.patch( @@ -29,6 +46,7 @@ def test_process_with_engine_iac(self, mock_runner_engine_iac): "tool": "engine_iac", "use_vulnerability_management": "true", "remote_config_repo": "test_repo", + "remote_config_branch": "" } config_tool = {"ENGINE_IAC": {"ENABLED": "true", "TOOL": "tool"}} secret_tool = "some_secret" @@ -39,7 +57,7 @@ def test_process_with_engine_iac(self, mock_runner_engine_iac): findings_list = ["finding1", "finding2"] input_core = InputCore( totalized_exclusions=[], - threshold_defined=Threshold, + threshold_defined=self.threshold, path_file_results="test/file", custom_message_break_build="message", scope_pipeline="pipeline", @@ -64,7 +82,11 @@ def test_process_with_engine_iac(self, mock_runner_engine_iac): self.assertEqual(result_input_core, input_core) self.secrets_manager_gateway.get_secret.assert_called_once_with(config_tool) mock_runner_engine_iac.assert_called_once_with( - dict_args, config_tool["ENGINE_IAC"]["TOOL"], secret_tool, self.devops_platform_gateway, "dev" + dict_args, + config_tool["ENGINE_IAC"]["TOOL"], + secret_tool, + self.devops_platform_gateway, + "dev", ) self.vulnerability_management.send_vulnerability_management.assert_called_once() self.vulnerability_management.get_findings_excepted.assert_called_once() @@ -78,6 +100,7 @@ def test_process_with_engine_iac_error(self, mock_runner_engine_iac): "tool": "engine_iac", "use_vulnerability_management": "true", "remote_config_repo": "test_repo", + "remote_config_branch": "" } config_tool = {"ENGINE_IAC": {"ENABLED": "true", "TOOL": "tool"}} @@ -94,10 +117,14 @@ def test_process_with_engine_iac_error(self, mock_runner_engine_iac): mock_runner_engine_iac.return_value = findings_list, input_core # Mock the send_vulnerability_management method - self.vulnerability_management.send_vulnerability_management.side_effect = ExceptionVulnerabilityManagement("Simulated error") + self.vulnerability_management.send_vulnerability_management.side_effect = ( + ExceptionVulnerabilityManagement("Simulated error") + ) # Mock the get_findings_excepted method - self.vulnerability_management.get_findings_excepted.side_effect = ExceptionFindingsExcepted("Simulated error") + self.vulnerability_management.get_findings_excepted.side_effect = ( + ExceptionFindingsExcepted("Simulated error") + ) # Call the process method result_findings_list, result_input_core = self.handle_scan.process( @@ -119,7 +146,8 @@ def test_process_with_engine_container(self, mock_runner_engine_container): "use_secrets_manager": "true", "tool": "engine_container", "remote_config_repo": "test_repo", - "use_vulnerability_management":"true", + "remote_config_branch": "", + "use_vulnerability_management": "true", } config_tool = {"ENGINE_CONTAINER": {"ENABLED": "true", "TOOL": "tool"}} secret_tool = {"token_prisma_cloud": "test"} @@ -129,13 +157,46 @@ def test_process_with_engine_container(self, mock_runner_engine_container): findings_list = ["finding1", "finding2"] input_core = InputCore( totalized_exclusions=[], - threshold_defined=Threshold, + threshold_defined=Threshold( + { + "VULNERABILITY": { + "Critical": 5, + "High": 8, + "Medium": 10, + "Low": 15, + }, + "COMPLIANCE": {"Critical": 1}, + "QUALITY_VULNERABILITY_MANAGEMENT": { + "PTS": [ + { + "PT1": { + "APPS": ["pipeline", "app2", "app3"], + "PROFILE": "STRONG", + } + }, + { + "PT2": { + "APPS": "ALL", + "PROFILE": "MODERATE", + } + }, + ], + "STRONG": {"Critical": 0, "High": 0, "Medium": 5, "Low": 15}, + "MODERATE": {"Critical": 1, "High": 3, "Medium": 5, "Low": 15}, + }, + } + ), path_file_results="test/file", custom_message_break_build="message", scope_pipeline="pipeline", stage_pipeline="Release", ) - mock_runner_engine_container.return_value = findings_list, input_core + component_list = [Component("component1", "version1"), Component("component2", "version2")] + + mock_runner_engine_container.return_value = findings_list, input_core, component_list + mock_product_type = Mock() + mock_product_type.name = "PT1" + self.vulnerability_management.get_product_type_service.return_value = mock_product_type # Call the process method result_findings_list, result_input_core = self.handle_scan.process( @@ -147,22 +208,64 @@ def test_process_with_engine_container(self, mock_runner_engine_container): self.assertEqual(result_input_core, input_core) self.secrets_manager_gateway.get_secret.assert_called_once_with(config_tool) - @mock.patch("builtins.print") - def test_process_with_engine_dast(self, mock_print): + @mock.patch("devsecops_engine_tools.engine_core.src.domain.usecases.handle_scan.runner_engine_dast") + @mock.patch("builtins.open", new_callable=mock.mock_open, read_data='''{ + "endpoint": "https://example.com", + "operations": [ + { + "operation": { + "headers": { + "accept": "/" + }, + "method": "POST", + "path": "/example_path", + "security_auth": { + "type": "jwt" + } + } + } + ] + }''') + def test_process_with_engine_dast(self, mock_open, mock_runner_engine_dast): dict_args = { - "use_secrets_manager": "false", + "use_secrets_manager": "true", "tool": "engine_dast", + "dast_file_path": "example_dast.json", + "use_vulnerability_management": "true", + "remote_config_repo": "dummie_repo" } - config_tool = {"ENGINE_DAST": "some_config"} - self.handle_scan.process(dict_args, config_tool) - mock_print.assert_called_once_with("not yet enabled") + secret_tool = {"github_token": "example_token"} + self.secrets_manager_gateway.get_secret.return_value = secret_tool + config_tool = {"ENGINE_DAST":{"ENABLED": "true", "TOOL": "NUCLEI"}} + input_core = InputCore( + totalized_exclusions=[], + threshold_defined=self.threshold, + path_file_results="test/file", + custom_message_break_build="message", + scope_pipeline="pipeline", + stage_pipeline="Release", + ) + # Simulates runner_engine_dast return + mock_runner_engine_dast.return_value = (["finding1", "finding2"], input_core) + # Call process method + result_findings_list, result_input_core = self.handle_scan.process(dict_args, config_tool) + # Verifies mock have been called correctly + mock_runner_engine_dast.assert_called_once_with( + dict_args, config_tool["ENGINE_DAST"], secret_tool, self.devops_platform_gateway + ) + # Verifica los resultados devueltos + self.assertEqual(result_findings_list, ["finding1", "finding2"]) + self.assertEqual(result_input_core, input_core) - @mock.patch("devsecops_engine_tools.engine_core.src.domain.usecases.handle_scan.runner_secret_scan") + @mock.patch( + "devsecops_engine_tools.engine_core.src.domain.usecases.handle_scan.runner_secret_scan" + ) def test_process_with_engine_secret(self, mock_runner_secret_scan): dict_args = { "use_secrets_manager": "true", "tool": "engine_secret", "remote_config_repo": "test_repo", + "remote_config_branch": "", "use_vulnerability_management": "true", } config_tool = {"ENGINE_SECRET": {"ENABLED": "true", "TOOL": "trufflehog"}} @@ -173,7 +276,44 @@ def test_process_with_engine_secret(self, mock_runner_secret_scan): findings_list = ["finding1", "finding2"] input_core = InputCore( totalized_exclusions=[], - threshold_defined=Threshold, + threshold_defined=self.threshold, + path_file_results="test/file", + custom_message_break_build="message", + scope_pipeline="pipeline", + stage_pipeline="Release", + ) + mock_runner_secret_scan.return_value = findings_list, input_core + + # Call the process method + result_findings_list, result_input_core = self.handle_scan.process( + dict_args, config_tool + ) + + # Assert the expected values + self.assertEqual(result_findings_list, findings_list) + self.assertEqual(result_input_core, input_core) + mock_runner_secret_scan.assert_called_once_with( + dict_args, config_tool["ENGINE_SECRET"]["TOOL"], self.devops_platform_gateway, secret_tool + ) + + @mock.patch("devsecops_engine_tools.engine_core.src.domain.usecases.handle_scan.runner_secret_scan") + def test_process_with_engine_secret_without_secret_manager(self, mock_runner_secret_scan): + dict_args = { + "use_secrets_manager": "true", + "tool": "engine_secret", + "remote_config_repo": "test_repo", + "remote_config_branch": "", + "use_vulnerability_management": "true", + } + config_tool = {"ENGINE_SECRET": {"ENABLED": "true", "TOOL": "trufflehog"}} + secret_tool = None + self.secrets_manager_gateway.get_secret.return_value = secret_tool + + # Mock the runner_engine_secret function and its return values + findings_list = ["finding1", "finding2"] + input_core = InputCore( + totalized_exclusions=[], + threshold_defined=self.threshold, path_file_results="test/file", custom_message_break_build="message", scope_pipeline="pipeline", @@ -199,6 +339,7 @@ def test_process_with_engine_secret_without_secret_manager(self, mock_runner_sec "use_secrets_manager": "false", "tool": "engine_secret", "remote_config_repo": "test_repo", + "remote_config_branch": "", "use_vulnerability_management": "true", } config_tool = {"ENGINE_SECRET": {"ENABLED": "true", "TOOL": "trufflehog"}} @@ -209,7 +350,7 @@ def test_process_with_engine_secret_without_secret_manager(self, mock_runner_sec findings_list = ["finding1", "finding2"] input_core = InputCore( totalized_exclusions=[], - threshold_defined=Threshold, + threshold_defined=self.threshold, path_file_results="test/file", custom_message_break_build="message", scope_pipeline="pipeline", @@ -237,11 +378,12 @@ def test_process_with_engine_dependencies(self, mock_runner_engine_dependencies) "use_secrets_manager": "true", "tool": "engine_dependencies", "remote_config_repo": "test_repo", - "use_vulnerability_management": "true" + "remote_config_branch": "", + "use_vulnerability_management": "true", } config_tool = { "ENGINE_DEPENDENCIES": "some_config", - "ENGINE_DEPENDENCIES": {"TOOL": "some_tool"} + "ENGINE_DEPENDENCIES": {"TOOL": "some_tool"}, } secret_tool = {"token_xray": "test"} self.secrets_manager_gateway.get_secret.return_value = secret_tool @@ -250,13 +392,13 @@ def test_process_with_engine_dependencies(self, mock_runner_engine_dependencies) findings_list = ["finding1", "finding2"] input_core = InputCore( totalized_exclusions=[], - threshold_defined=Threshold, + threshold_defined=self.threshold, path_file_results="test/file", custom_message_break_build="message", scope_pipeline="pipeline", stage_pipeline="Release", ) - mock_runner_engine_dependencies.return_value = findings_list, input_core + mock_runner_engine_dependencies.return_value = findings_list, input_core, None # Call the process method result_findings_list, result_input_core = self.handle_scan.process( @@ -268,7 +410,5 @@ def test_process_with_engine_dependencies(self, mock_runner_engine_dependencies) self.assertEqual(result_input_core, input_core) self.secrets_manager_gateway.get_secret.assert_called_once_with(config_tool) mock_runner_engine_dependencies.assert_called_once_with( - dict_args, config_tool, secret_tool, self.devops_platform_gateway + dict_args, config_tool, secret_tool, self.devops_platform_gateway, self.sbom_gateway ) - - diff --git a/tools/devsecops_engine_tools/engine_dast/src/deployment/infrastructure/.gitkeep b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/__init__.py similarity index 100% rename from tools/devsecops_engine_tools/engine_dast/src/deployment/infrastructure/.gitkeep rename to tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/__init__.py diff --git a/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/aws/test_s3_manager.py b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/aws/test_s3_manager.py index a81df5678..b32803a3f 100644 --- a/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/aws/test_s3_manager.py +++ b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/aws/test_s3_manager.py @@ -24,6 +24,7 @@ def test_send_metrics(self, mock_assume_role , mock_client): config_tool = { "METRICS_MANAGER": { "AWS": { + "USE_ROLE": True, "ROLE_ARN": "arn:aws:iam::123456789012:role/MyRole", "REGION_NAME": "us-west-2", "BUCKET": "my-bucket", @@ -46,6 +47,39 @@ def test_send_metrics(self, mock_assume_role , mock_client): aws_session_token=mock.ANY, ) date = datetime.datetime.now() + mock_client.return_value.put_object.assert_called_once_with( + Bucket="my-bucket", Key=f"engine_tools/my-tool/{date.strftime('%Y')}/{date.strftime('%m')}/{date.strftime('%d')}/file.txt", Body=mock.ANY + ) + + @patch("boto3.session.Session.client") + def test_send_metrics_without_role(self, mock_client): + # Mock the necessary dependencies + mock_client.return_value = MagicMock() + + # Set up test data + config_tool = { + "METRICS_MANAGER": { + "AWS": { + "USE_ROLE": False, + "ROLE_ARN": "arn:aws:iam::123456789012:role/MyRole", + "REGION_NAME": "us-ueast-2", + "BUCKET": "my-bucket", + } + } + } + tool = "my-tool" + file_path = "/path/to/my/file.txt" + + with mock.patch("builtins.open", create=True) as mock_open: + # Call the method under test + self.s3_manager.send_metrics(config_tool, tool, file_path) + + # Assert that the necessary methods were called with the correct arguments + mock_client.assert_called_once_with( + service_name="s3", + region_name="us-ueast-2" + ) + date = datetime.datetime.now() mock_client.return_value.put_object.assert_called_once_with( Bucket="my-bucket", Key=f"engine_tools/my-tool/{date.strftime('%Y')}/{date.strftime('%m')}/{date.strftime('%d')}/file.txt", Body=mock.ANY ) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/aws/test_secrets_manager.py b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/aws/test_secrets_manager.py index 56fa84d03..983e7b2bd 100644 --- a/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/aws/test_secrets_manager.py +++ b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/aws/test_secrets_manager.py @@ -21,6 +21,7 @@ def test_get_secret_with_execution_different_account( config_tool = { "SECRET_MANAGER": { "AWS": { + "USE_ROLE": True, "ROLE_ARN": "arn:aws:iam::123456789012:role/MyRole", "REGION_NAME": "us-west-2", "SECRET_NAME": "my-secret-different-account", @@ -50,3 +51,35 @@ def test_get_secret_with_execution_different_account( mock_client.return_value.get_secret_value.assert_called_once_with( SecretId="my-secret-different-account" ) + + @patch("boto3.session.Session.client") + def test_get_secret_without_role( + self, mock_client + ): + mock_client.return_value = MagicMock() + + config_tool = { + "SECRET_MANAGER": { + "AWS": { + "USE_ROLE": False, + "ROLE_ARN": "arn:aws:iam::123456789012:role/MyRole", + "REGION_NAME": "us-west-2", + "SECRET_NAME": "my-secret-different-account", + } + } + } + + mock_client.return_value.get_secret_value.return_value = { + "SecretString": '{"username": "admin", "password": "password123"}' + } + + secret = self.secrets_manager.get_secret(config_tool) + + assert secret == {"username": "admin", "password": "password123"} + mock_client.assert_called_once_with( + service_name="secretsmanager", + region_name="us-west-2", + ) + mock_client.return_value.get_secret_value.assert_called_once_with( + SecretId="my-secret-different-account" + ) diff --git a/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/defect_dojo/test_defect_dojo.py b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/defect_dojo/test_defect_dojo.py index 03fd8a53e..b5a129f09 100644 --- a/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/defect_dojo/test_defect_dojo.py +++ b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/defect_dojo/test_defect_dojo.py @@ -8,6 +8,13 @@ from devsecops_engine_tools.engine_core.src.domain.model.vulnerability_management import ( VulnerabilityManagement, ) +from devsecops_engine_tools.engine_core.src.domain.model.component import Component +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.models.engagement import ( + Engagement, +) +from devsecops_engine_tools.engine_core.src.domain.model.customs_exceptions import ( + ExceptionVulnerabilityManagement, +) class TestDefectDojoPlatform(unittest.TestCase): @@ -23,6 +30,7 @@ def test_send_vulnerability_management(self, mock_send_import_scan): "token_vulnerability_management": "token1", "token_cmdb": "token2", "tool": "engine_iac", + "platform": ["k8s"], } self.vulnerability_management.secret_tool = { "token_defect_dojo": "token3", @@ -35,11 +43,30 @@ def test_send_vulnerability_management(self, mock_send_import_scan): "VULNERABILITY_MANAGER": { "BRANCH_FILTER": "trunk,master,release,develop", "DEFECT_DOJO": { - "CMDB_MAPPING_PATH": "mapping_path", - "HOST_CMDB": "cmdb_host", - "REGEX_EXPRESSION_CMDB": "regex", "HOST_DEFECT_DOJO": "host_defect_dojo", "MAX_RETRIES_QUERY": 5, + "CMDB": { + "USE_CMDB": True, + "HOST_CMDB": "cmdb_host", + "REGEX_EXPRESSION_CMDB": "regex", + "CMDB_MAPPING_PATH": "mapping_path", + "CMDB_MAPPING": { + "PRODUCT_TYPE_NAME": "nombreevc", + "PRODUCT_NAME": "nombreapp", + "TAG_PRODUCT": "nombreentorno", + "PRODUCT_DESCRIPTION": "arearesponsableti", + "CODIGO_APP": "CodigoApp", + }, + "CMDB_REQUEST_RESPONSE": { + "HEADERS": { + "Content-Type": "application/json", + "tokenkey": "tokenvalue", + }, + "METHOD": "POST", + "BODY": {"codapp": "codappvalue"}, + "RESPONSE": [0], + }, + }, }, } } @@ -77,6 +104,15 @@ def test_send_vulnerability_management(self, mock_send_import_scan): personal_access_token="access_token", token_cmdb="token2", host_cmdb="cmdb_host", + cmdb_request_response={ + "HEADERS": { + "Content-Type": "application/json", + "tokenkey": "tokenvalue", + }, + "METHOD": "POST", + "BODY": {"codapp": "codappvalue"}, + "RESPONSE": [0], + }, expression="regex", token_defect_dojo="token1", host_defect_dojo="host_defect_dojo", @@ -90,7 +126,7 @@ def test_send_vulnerability_management(self, mock_send_import_scan): branch_tag="trunk", commit_hash="commit_hash", environment="Development", - tags="engine_iac", + tags="engine_iac_k8s", ) def test_send_vulnerability_management_exception(self): @@ -110,13 +146,274 @@ def test_send_vulnerability_management_exception(self): in str(context.exception) ) + def test_build_request_with_cmdb(self): + use_cmdb = True + tags = "engine_iac_k8s" + + self.vulnerability_management.scan_type = "CHECKOV" + self.vulnerability_management.input_core = MagicMock() + self.vulnerability_management.input_core.path_file_results = "file_path" + self.vulnerability_management.input_core.scope_pipeline = "engagement_name" + self.vulnerability_management.source_code_management_uri = "source_code_uri" + self.vulnerability_management.version = "1.0" + self.vulnerability_management.build_id = "build_id" + self.vulnerability_management.branch_tag = "trunk" + self.vulnerability_management.commit_hash = "commit_hash" + self.vulnerability_management.environment = "dev" + + self.vulnerability_management.config_tool = { + "VULNERABILITY_MANAGER": { + "BRANCH_FILTER": "trunk,master,release,develop", + "DEFECT_DOJO": { + "HOST_DEFECT_DOJO": "host_defect_dojo", + "MAX_RETRIES_QUERY": 5, + "CMDB": { + "USE_CMDB": True, + "HOST_CMDB": "cmdb_host", + "REGEX_EXPRESSION_CMDB": "regex", + "CMDB_MAPPING_PATH": "mapping_path", + "CMDB_MAPPING": { + "PRODUCT_TYPE_NAME": "nombreevc", + "PRODUCT_NAME": "nombreapp", + "TAG_PRODUCT": "nombreentorno", + "PRODUCT_DESCRIPTION": "arearesponsableti", + "CODIGO_APP": "CodigoApp", + }, + "CMDB_REQUEST_RESPONSE": { + "HEADERS": { + "Content-Type": "application/json", + "tokenkey": "tokenvalue", + }, + "METHOD": "POST", + "BODY": {"codapp": "codappvalue"}, + "RESPONSE": [0], + }, + }, + }, + } + } + self.vulnerability_management.base_compact_remote_config_url = ( + "http://example.com/" + ) + self.vulnerability_management.access_token = "access_token" + + self.token_cmdb = "token_cmdb" + self.token_dd = "token_dd" + + self.scan_type_mapping = {"CHECKOV": "Checkov Scan"} + self.enviroment_mapping = { + "dev": "Development", + "qa": "Staging", + "pdn": "Production", + "default": "Production", + } + + with patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.Connect.cmdb" + ) as mock_cmdb: + mock_cmdb.return_value = "cmdb_request_result" + + result = self.defect_dojo._build_request_importscan( + vulnerability_management=self.vulnerability_management, + token_cmdb=self.token_cmdb, + token_dd=self.token_dd, + scan_type_mapping=self.scan_type_mapping, + enviroment_mapping=self.enviroment_mapping, + tags=tags, + use_cmdb=use_cmdb, + ) + + mock_cmdb.assert_called_once_with( + cmdb_mapping={ + "product_type_name": "nombreevc", + "product_name": "nombreapp", + "tag_product": "nombreentorno", + "product_description": "arearesponsableti", + "codigo_app": "CodigoApp", + }, + compact_remote_config_url="http://example.com/mapping_path", + personal_access_token="access_token", + token_cmdb=self.token_cmdb, + host_cmdb="cmdb_host", + cmdb_request_response={ + "HEADERS": { + "Content-Type": "application/json", + "tokenkey": "tokenvalue", + }, + "METHOD": "POST", + "BODY": {"codapp": "codappvalue"}, + "RESPONSE": [0], + }, + scan_type="Checkov Scan", + file="file_path", + engagement_name="engagement_name", + source_code_management_uri="source_code_uri", + tags=tags, + version="1.0", + build_id="build_id", + branch_tag="trunk", + commit_hash="commit_hash", + service="engagement_name", + environment="Development", + token_defect_dojo=self.token_dd, + host_defect_dojo="host_defect_dojo", + expression="regex", + ) + self.assertEqual(result, "cmdb_request_result") + + def test_build_request_without_cmdb(self): + use_cmdb = False + tags = "engine_iac_k8s" + + self.vulnerability_management.scan_type = "CHECKOV" + self.vulnerability_management.input_core = MagicMock() + self.vulnerability_management.input_core.path_file_results = "file_path" + self.vulnerability_management.input_core.scope_pipeline = "engagement_name" + self.vulnerability_management.source_code_management_uri = "source_code_uri" + self.vulnerability_management.version = "1.0" + self.vulnerability_management.build_id = "build_id" + self.vulnerability_management.branch_tag = "trunk" + self.vulnerability_management.commit_hash = "commit_hash" + self.vulnerability_management.environment = "dev" + self.vulnerability_management.vm_product_type_name = "ProductType" + self.vulnerability_management.vm_product_name = "ProductName" + self.vulnerability_management.vm_product_description = "ProductDescription" + + self.vulnerability_management.config_tool = { + "VULNERABILITY_MANAGER": { + "BRANCH_FILTER": "trunk,master,release,develop", + "DEFECT_DOJO": { + "HOST_DEFECT_DOJO": "host_defect_dojo", + "MAX_RETRIES_QUERY": 5, + "CMDB": {"USE_CMDB": True, "REGEX_EXPRESSION_CMDB": "regex"}, + }, + } + } + self.vulnerability_management.base_compact_remote_config_url = ( + "http://example.com/" + ) + self.vulnerability_management.access_token = "access_token" + + self.token_cmdb = "token_cmdb" + self.token_dd = "token_dd" + + self.scan_type_mapping = {"CHECKOV": "Checkov Scan"} + self.enviroment_mapping = { + "dev": "Development", + "qa": "Staging", + "pdn": "Production", + "default": "Production", + } + + with patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.ImportScanSerializer" + ) as mock_serializer: + mock_serializer().load.return_value = "import_scan_request_result" + + result = self.defect_dojo._build_request_importscan( + vulnerability_management=self.vulnerability_management, + token_cmdb=self.token_cmdb, + token_dd=self.token_dd, + scan_type_mapping=self.scan_type_mapping, + enviroment_mapping=self.enviroment_mapping, + tags=tags, + use_cmdb=use_cmdb, + ) + + mock_serializer().load.assert_called_once_with( + { + "product_type_name": "ProductType", + "product_name": "ProductName", + "product_description": "ProductDescription", + "code_app": "ProductName", + "scan_type": "Checkov Scan", + "file": "file_path", + "engagement_name": "engagement_name", + "source_code_management_uri": "source_code_uri", + "tags": tags, + "version": "1.0", + "build_id": "build_id", + "branch_tag": "trunk", + "commit_hash": "commit_hash", + "service": "engagement_name", + "environment": "Development", + "token_defect_dojo": self.token_dd, + "host_defect_dojo": "host_defect_dojo", + "expression": "regex", + } + ) + self.assertEqual(result, "import_scan_request_result") + + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.SessionManager" + ) + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.Product.get_product" + ) + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.Connect.get_code_app" + ) + def test_get_product_type_service( + self, cmdb_code, mock_product, mock_session_manager + ): + service = "test" + dict_args = {"token_vulnerability_management": "token1"} + secret_tool = None + config_tool = { + "VULNERABILITY_MANAGER": { + "DEFECT_DOJO": { + "HOST_DEFECT_DOJO": "host_defect_dojo", + "LIMITS_QUERY": 80, + "MAX_RETRIES_QUERY": 5, + "CMDB": {"REGEX_EXPRESSION_CMDB": "regex"}, + } + } + } + + mock_session_manager.return_value = MagicMock() + + cmdb_code.return_value = "CodigoApp" + + product_list = [ + MagicMock( + results=[ + MagicMock( + id=1, + name="name1", + prod_type=35, + ), + ], + prefetch=MagicMock(), + ) + ] + mock_product.side_effect = product_list + + result = self.defect_dojo.get_product_type_service( + service, dict_args, secret_tool, config_tool + ) + + mock_session_manager.assert_called_with("token1", "host_defect_dojo") + self.assertIsNotNone(result) + + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.DefectDojoPlatform._date_reason_based" + ) + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.FindingExclusion.get_finding_exclusion" + ) @patch( "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.SessionManager" ) @patch( "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.Finding.get_finding" ) - def test_get_findings_excepted(self, mock_finding, mock_session_manager): + def test_get_findings_excepted( + self, + mock_finding, + mock_session_manager, + mock_finding_exclusion, + mock_date_reason_based, + ): service = "test" dict_args = {"tool": "engine_iac", "token_vulnerability_management": "token1"} secret_tool = {"token_defect_dojo": "token2"} @@ -172,6 +469,21 @@ def test_get_findings_excepted(self, mock_finding, mock_session_manager): ), ] ), + # Findings out of scope + MagicMock( + results=[ + MagicMock( + vuln_id_from_tool="id1", + file_path="path1", + last_status_update="2024-01-10T00:00:00Z", + ), + MagicMock( + vuln_id_from_tool="id2", + file_path="path2", + last_status_update="2024-01-10T00:00:00Z", + ), + ] + ), # Findings Transferred Finding MagicMock( results=[ @@ -193,8 +505,76 @@ def test_get_findings_excepted(self, mock_finding, mock_session_manager): ), ] ), + # Findings Whitelist + MagicMock( + results=[ + MagicMock(vuln_id_from_tool="CVE-2024-0001", file_path="path1"), + MagicMock(vuln_id_from_tool="CVE-2024-0002", file_path="path2"), + ] + ), + ] + mock_finding.return_value.results = findings_list + + findings_exclusion_list = [ + MagicMock( + uuid="id1", + unique_id_from_tool="CVE-2024-0001", + type="white_list", + create_date="2024-02-21T00:00:00Z", + expiration_date="2024-02-29T00:00:00Z", + ), + MagicMock( + uuid="id2", + unique_id_from_tool="CVE-2024-0002", + type="white_list", + create_date="2024-02-21T00:00:00Z", + expiration_date="2024-02-29T00:00:00Z", + ), + ] + mock_finding_exclusion.return_value.results = findings_exclusion_list + + mock_date_reason_based.side_effect = [ + ( + "10012024", + "10042024", + ), + ( + "15012024", + "10062024", + ), + ( + "10062024", + "", + ), + ( + "10062024", + "", + ), + ( + "10012024", + "", + ), + ( + "10012024", + "", + ), + ( + "14082024", + "15082024", + ), + ( + "14082024", + "15082024", + ), + ( + "21022024", + "29022024", + ), + ( + "21022024", + "29022024", + ), ] - mock_finding.side_effect = findings_list result = self.defect_dojo.get_findings_excepted( service, dict_args, secret_tool, config_tool @@ -215,22 +595,42 @@ def test_get_findings_excepted(self, mock_finding, mock_session_manager): Exclusions( id="id2", where="path2", create_date="10062024", expired_date="" ), + Exclusions( + id="id1", where="path1", create_date="10012024", expired_date="" + ), + Exclusions( + id="id2", where="path2", create_date="10012024", expired_date="" + ), Exclusions( id="id3", where="pathq", create_date="14082024", expired_date="15082024" ), Exclusions( id="id4", where="path2", create_date="14082024", expired_date="15082024" ), + Exclusions( + id="id1", where="path1", create_date="21022024", expired_date="29022024" + ), + Exclusions( + id="id2", where="path2", create_date="21022024", expired_date="29022024" + ), ] self.assertEqual(result, expected_result) + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.FindingExclusion.get_finding_exclusion" + ) @patch( "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.SessionManager" ) @patch( "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.Finding.get_finding" ) - def test_get_findings_excepted_sca(self, mock_finding, mock_session_manager): + def test_get_findings_excepted_sca( + self, + mock_finding, + mock_session_manager, + mock_finding_exclusion, + ): service = "test" dict_args = { "tool": "engine_dependencies", @@ -280,11 +680,17 @@ def test_get_findings_excepted_sca(self, mock_finding, mock_session_manager): ), # Findings false positive MagicMock(results=[]), + # Findings out of scope + MagicMock(results=[]), # Findings Transferred Finding MagicMock(results=[]), + # Findings Whitelist + MagicMock(results=[]), ] mock_finding.side_effect = findings_list + mock_finding_exclusion.return_value.results = [] + result = self.defect_dojo.get_findings_excepted( service, dict_args, secret_tool, config_tool ) @@ -293,7 +699,7 @@ def test_get_findings_excepted_sca(self, mock_finding, mock_session_manager): mock_finding.assert_called_with( session=mock_session_manager.return_value, service=service, - risk_status="Transfer Accepted", + risk_status="On Whitelist", tags="engine_dependencies", limit=80, ) @@ -356,7 +762,7 @@ def test_get_findings_excepted_exception(self): ) @patch( - "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.DefectDojoPlatform._format_date_to_dd_format" + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.FindingExclusion.get_finding_exclusion" ) @patch( "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.SessionManager" @@ -368,7 +774,11 @@ def test_get_findings_excepted_exception(self): "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.DefectDojoPlatform._get_report_exclusions" ) def test_get_all( - self, mock_exclusions, mock_finding, mock_session_manager, mock_format_date + self, + mock_exclusions, + mock_finding, + mock_session_manager, + mock_finding_exclusion, ): service = "test" dict_args = { @@ -441,6 +851,25 @@ def test_get_all( ), ] mock_finding.return_value.results = findings_list + + findings_exclusion_list = [ + MagicMock( + uuid="id1", + unique_id_from_tool="CVE-2024-0001", + type="white_list", + create_date="2024-02-21T00:00:00Z", + expiration_date="2024-02-29T00:00:00Z", + ), + MagicMock( + uuid="id2", + unique_id_from_tool="CVE-2024-0002", + type="white_list", + create_date="2024-02-21T00:00:00Z", + expiration_date="2024-02-29T00:00:00Z", + ), + ] + mock_finding_exclusion.return_value.results = findings_exclusion_list + expected_result = [ Report( id="id2", @@ -480,6 +909,7 @@ def test_get_all( session=mock_session_manager.return_value, service=service, limit=80, + duplicate="false", ) mock_exclusions.assert_called_once() assert exclusions == mock_exclusions.return_value @@ -506,7 +936,14 @@ def test_get_all_findings_exception(self): def test_get_active_engagements(self, mock_engagement, mock_import_scan_request): dict_args = {"token_vulnerability_management": "token1"} secret_tool = MagicMock() - config_tool = {"VULNERABILITY_MANAGER": {"DEFECT_DOJO": {"HOST_DEFECT_DOJO": "host_defect_dojo", "LIMITS_QUERY": 999}}} + config_tool = { + "VULNERABILITY_MANAGER": { + "DEFECT_DOJO": { + "HOST_DEFECT_DOJO": "host_defect_dojo", + "LIMITS_QUERY": 999, + } + } + } engagement_name = "engagement_name" mock_engagement.get_engagements.return_value = MagicMock() @@ -520,21 +957,28 @@ def test_get_active_engagements(self, mock_engagement, mock_import_scan_request) def test_get_active_engagements_exception(self): dict_args = {"token_vulnerability_management": "token1"} secret_tool = MagicMock() - config_tool = {"VULNERABILITY_MANAGER": {"DEFECT_DOJO": {"HOST_DEFECT_DOJO": "host_defect_dojo", "LIMITS_QUERY": 999}}} + config_tool = { + "VULNERABILITY_MANAGER": { + "DEFECT_DOJO": { + "HOST_DEFECT_DOJO": "host_defect_dojo", + "LIMITS_QUERY": 999, + } + } + } engagement_name = "engagement_name" with unittest.TestCase().assertRaises(Exception) as context: self.defect_dojo.get_active_engagements( - engagement_name, dict_args, secret_tool, config_tool - ) + engagement_name, dict_args, secret_tool, config_tool + ) assert "Error getting engagements with the following error:" in str( context.exception ) @patch( - "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.DefectDojoPlatform._create_exclusion" + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.DefectDojoPlatform._create_report_exclusion" ) - def test_get_report_exclusions(self, mock_create_exclusion): + def test_get_report_exclusions(self, mock_create_report_exclusion): total_findings = [ MagicMock( risk_accepted=True, @@ -546,16 +990,230 @@ def test_get_report_exclusions(self, mock_create_exclusion): MagicMock( risk_accepted=None, false_p=None, + out_of_scope=None, risk_status="Transfer Accepted", ), MagicMock( risk_accepted=None, + out_of_scope=True, false_p=None, risk_status=None, ), + MagicMock( + risk_accepted=None, + out_of_scope=None, + false_p=None, + risk_status="On Whitelist", + vuln_id_from_tool="CVE-2024-0001", + ), ] date_fn = MagicMock() + host_dd = "host_defect_dojo" + + exclusions = self.defect_dojo._get_report_exclusions( + total_findings, date_fn, host_dd + ) + + assert len(exclusions) == 5 + + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.Engagement" + ) + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.SessionManager" + ) + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.Component" + ) + def test_send_sbom_components_success( + self, mock_component, mock_session_manager, mock_engagement + ): + # Configurar los mocks + mock_engagement.get_engagements.return_value.results = [ + Engagement(id=1, name="test_service") + ] + mock_session_manager.return_value = MagicMock() + + mock_component.get_component.return_value.results = [] + mock_component.create_component.return_value = Component( + name="component_name", version="1.0" + ) + + # Datos de prueba + sbom_components = [ + Component(name="component1", version="1.0"), + Component(name="component2", version="2.0"), + ] + service = "test_service" + dict_args = {"token_vulnerability_management": "test_token"} + secret_tool = {"token_defect_dojo": "secret_token"} + config_tool = { + "VULNERABILITY_MANAGER": { + "DEFECT_DOJO": { + "HOST_DEFECT_DOJO": "http://defectdojo", + "MAX_RETRIES_QUERY": 3, + "LIMITS_QUERY": 100, + } + } + } + + # Llamar a la función + self.defect_dojo.send_sbom_components( + sbom_components, service, dict_args, secret_tool, config_tool + ) + + # Verificar que se llamaron las funciones esperadas + mock_session_manager.assert_called_once() + mock_engagement.get_engagements.assert_called_once() + assert mock_component.get_component.call_count == 2 + assert mock_component.create_component.call_count == 2 - exclusions = self.defect_dojo._get_report_exclusions(total_findings, date_fn) + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.Engagement" + ) + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo.SessionManager" + ) + def test_send_sbom_components_exception( + self, mock_session_manager, mock_engagement + ): + # Configurar los mocks + mock_engagement.get_engagements.side_effect = Exception("Test exception") + + # Datos de prueba + sbom_components = [Component(name="component1", version="1.0")] + service = "test_service" + dict_args = {"token_vulnerability_management": "test_token"} + secret_tool = {"token_defect_dojo": "secret_token"} + config_tool = { + "VULNERABILITY_MANAGER": { + "DEFECT_DOJO": { + "HOST_DEFECT_DOJO": "http://defectdojo", + "MAX_RETRIES_QUERY": 3, + "LIMITS_QUERY": 100, + } + } + } + + # Verificar que se lanza la excepción esperada + with self.assertRaises(ExceptionVulnerabilityManagement): + self.defect_dojo.send_sbom_components( + sbom_components, service, dict_args, secret_tool, config_tool + ) + + def test_date_reason_based_false_positive(self): + finding = MagicMock() + finding.last_status_update = "2024-01-10T00:00:00Z" + date_fn = MagicMock(return_value="10012024") + reason = self.defect_dojo.FALSE_POSITIVE + tool = "engine_risk" + + create_date, expired_date = self.defect_dojo._date_reason_based( + finding, date_fn, reason, tool + ) + + self.assertEqual(create_date, "10012024") + self.assertEqual(expired_date, date_fn(None)) + + def test_date_reason_based_out_of_scope(self): + finding = MagicMock() + finding.last_status_update = "2024-01-10T00:00:00Z" + date_fn = MagicMock(return_value="10012024") + reason = self.defect_dojo.OUT_OF_SCOPE + tool = "engine_risk" + + create_date, expired_date = self.defect_dojo._date_reason_based( + finding, date_fn, reason, tool + ) + + self.assertEqual(create_date, "10012024") + self.assertEqual(expired_date, date_fn(None)) + + def test_date_reason_based_transferred_finding(self): + finding = MagicMock() + finding.transfer_finding.date = "2024-08-14" + finding.transfer_finding.expiration_date = "2024-08-15T00:00:00Z" + date_fn = MagicMock(side_effect=["14082024", "15082024"]) + reason = self.defect_dojo.TRANSFERRED_FINDING + tool = "engine_risk" + + create_date, expired_date = self.defect_dojo._date_reason_based( + finding, date_fn, reason, tool + ) + + self.assertEqual(create_date, "14082024") + self.assertEqual(expired_date, "15082024") + + def test_date_reason_based_risk_accepted(self): + finding = MagicMock() + finding.accepted_risks = [ + { + "created": "2024-01-10T00:00:00Z", + "expiration_date": "2024-04-10T00:00:00Z", + } + ] + date_fn = MagicMock(side_effect=["10012024", "10042024"]) + reason = self.defect_dojo.RISK_ACCEPTED + tool = "engine_risk" + + create_date, expired_date = self.defect_dojo._date_reason_based( + finding, date_fn, reason, tool + ) + + self.assertEqual(create_date, "10012024") + self.assertEqual(expired_date, "10042024") + + def test_date_reason_based_on_whitelist_engine_risk(self): + finding = MagicMock() + finding.vuln_id_from_tool = "CVE-2024-0001" + date_fn = MagicMock(side_effect=["21022024", "29022024"]) + reason = self.defect_dojo.ON_WHITELIST + tool = "engine_risk" + white_list = [ + MagicMock( + unique_id_from_tool="CVE-2024-0001", + create_date="2024-02-21T00:00:00Z", + expiration_date="2024-02-29T00:00:00Z", + ) + ] + + create_date, expired_date = self.defect_dojo._date_reason_based( + finding, date_fn, reason, tool, white_list=white_list + ) + + self.assertEqual(create_date, "21022024") + self.assertEqual(expired_date, "29022024") + + def test_date_reason_based_on_whitelist(self): + finding = MagicMock() + finding.vulnerability_ids = [{"vulnerability_id": "CVE-2024-0001"}] + date_fn = MagicMock(side_effect=["21022024", "29022024"]) + reason = self.defect_dojo.ON_WHITELIST + tool = "engine_container" + white_list = [ + MagicMock( + unique_id_from_tool="CVE-2024-0001", + create_date="2024-02-21T00:00:00Z", + expiration_date="2024-02-29T00:00:00Z", + ) + ] + + create_date, expired_date = self.defect_dojo._date_reason_based( + finding, date_fn, reason, tool, white_list=white_list + ) + + self.assertEqual(create_date, "21022024") + self.assertEqual(expired_date, "29022024") + + def test_date_reason_based_default(self): + finding = MagicMock() + date_fn = MagicMock(return_value="default_date") + reason = "UNKNOWN_REASON" + tool = "engine_risk" + + create_date, expired_date = self.defect_dojo._date_reason_based( + finding, date_fn, reason, tool + ) - assert len(exclusions) == 3 + self.assertEqual(create_date, "default_date") + self.assertEqual(expired_date, "default_date") diff --git a/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/printer_pretty_table/test_printer_pretty_table.py b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/printer_pretty_table/test_printer_pretty_table.py index ca3f46d26..30919a91a 100644 --- a/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/printer_pretty_table/test_printer_pretty_table.py +++ b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/printer_pretty_table/test_printer_pretty_table.py @@ -103,7 +103,16 @@ def test_print_table_without_findings(self, mock_print): @patch("builtins.print") def test_print_table_exclusions(self, mock_print): # Arrange - exclusions = [{"severity": "severity" ,"id": "id", "where": "path", "create_date": "01042023", "expired_date": "04032023", "reason": "reason"}] + exclusions = [ + { + "severity": "severity", + "id": "id", + "where": "path", + "create_date": "01042023", + "expired_date": "04032023", + "reason": "reason", + } + ] printer = PrinterPrettyTable() # Act @@ -119,13 +128,14 @@ def test_print_table_report(self, mock_print): report_list = [ Report( risk_score=1, - id="id2", - date="21022024", + vm_id="id1 id2", + vm_id_url="url1 url2", status="stat2", where="path", tags=["tag1"], severity="low", active=True, + service="service1", ), ] printer = PrinterPrettyTable() @@ -135,6 +145,25 @@ def test_print_table_report(self, mock_print): # Assert assert mock_print.called - # Add more assertions to validate the output + @patch("builtins.print") + def test_print_table_report_exlusions(self, mock_print): + # Arrange + exclusions = [ + { + "vm_id": "id", + "vm_id_url": "url", + "tags": ["tag1"], + "service": "service1", + "create_date": "01042023", + "expired_date": "04032023", + "reason": "reason", + } + ] + printer = PrinterPrettyTable() + + # Act + printer.print_table_report_exlusions(exclusions) + # Assert + assert mock_print.called diff --git a/tools/devsecops_engine_tools/engine_dast/src/domain/model/.gitkeep b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/printer_rich_table/__init__.py similarity index 100% rename from tools/devsecops_engine_tools/engine_dast/src/domain/model/.gitkeep rename to tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/printer_rich_table/__init__.py diff --git a/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/printer_rich_table/test_printer_rich_table.py b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/printer_rich_table/test_printer_rich_table.py new file mode 100644 index 000000000..9764090ce --- /dev/null +++ b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/printer_rich_table/test_printer_rich_table.py @@ -0,0 +1,69 @@ +from unittest.mock import patch +from devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.printer_rich_table.printer_rich_table import ( + PrinterRichTable, +) +from devsecops_engine_tools.engine_core.src.domain.model.finding import Finding +from devsecops_engine_tools.engine_core.src.domain.model.report import Report + + +class TestPrinterRichTable: + def test_print_table_findings(self): + finding_list = [ + Finding( + id="1", + cvss="7.8", + where="Location 1", + description="Description 1", + severity="high", + identification_date="2021-01-01", + published_date_cve=None, + module="engine_iac", + category="vulnerability", + requirements="Requirement 1", + tool="Tool 1", + ) + ] + printer = PrinterRichTable() + + result = printer.print_table_findings(finding_list) + + assert result is None + + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.printer_rich_table.printer_rich_table.Console" + ) + def test_print_table_report(self, mock_console): + report_list = [ + Report( + risk_score=7.8, + vm_id="1 2", + tags=["tag1"], + service="service1 service2", + ) + ] + printer = PrinterRichTable() + + printer.print_table_report(report_list) + + mock_console().print.assert_called_once() + + @patch( + "devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.printer_rich_table.printer_rich_table.Console" + ) + def test_print_table_exclusions(self, mock_console): + exclusions_list = [ + { + "vm_id": "1 2", + "tags": ["tag1"], + "service": "service1 service2", + "create_date": "01012021", + "expired_date": "02012021", + "reason": "reason1", + "vm_id_url": "url1", + } + ] + printer = PrinterRichTable() + + printer.print_table_exclusions(exclusions_list) + + mock_console().print.assert_called_once() diff --git a/tools/devsecops_engine_tools/engine_dast/src/domain/usecases/.gitkeep b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/syft/__init__.py similarity index 100% rename from tools/devsecops_engine_tools/engine_dast/src/domain/usecases/.gitkeep rename to tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/syft/__init__.py diff --git a/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/syft/test_syft.py b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/syft/test_syft.py new file mode 100644 index 000000000..4e95483cd --- /dev/null +++ b/tools/devsecops_engine_tools/engine_core/test/infrastructure/driven_adapters/syft/test_syft.py @@ -0,0 +1,211 @@ +import unittest +from unittest.mock import patch, MagicMock +from devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft import Syft +from devsecops_engine_tools.engine_core.src.domain.model.component import Component + +class TestSyft(unittest.TestCase): + + def _init_get_components(mock_platform, mock_install, mock_run_syft, mock_get_list_component, os_platform, command_prefix): + mock_platform.return_value = os_platform + mock_install.return_value = command_prefix + mock_run_syft.return_value = "result_sbom.json" + mock_get_list_component.return_value = [Component(name="component1", version="1.0.0"), Component(name="component2", version="2.0.0")] + + syft = Syft() + artifact = "artifact" + config = { + "SYFT": { + "SYFT_VERSION": "0.30.1", + "OUTPUT_FORMAT": "json" + } + } + service_name = "test_service" + + return syft.get_components(artifact, config, service_name) + + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.get_list_component') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.Syft._run_syft') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.Syft._install_tool_unix') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.platform.system') + def test_get_components_linux(self, mock_platform, mock_install_unix, mock_run_syft, mock_get_list_component): + components = TestSyft._init_get_components(mock_platform, mock_install_unix, mock_run_syft, mock_get_list_component, "Linux", "./syft") + + self.assertEqual(mock_install_unix.call_count, 1) + self.assertEqual(len(components), 2) + self.assertEqual(components[0].name, "component1") + self.assertEqual(components[1].name, "component2") + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.get_list_component') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.Syft._run_syft') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.Syft._install_tool_unix') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.platform.system') + def test_get_components_darwin(self, mock_platform, mock_install_unix, mock_run_syft, mock_get_list_component): + components = TestSyft._init_get_components(mock_platform, mock_install_unix, mock_run_syft, mock_get_list_component, "Darwin", "./syft") + + self.assertEqual(mock_install_unix.call_count, 1) + self.assertEqual(len(components), 2) + self.assertEqual(components[0].name, "component1") + self.assertEqual(components[1].name, "component2") + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.get_list_component') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.Syft._run_syft') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.Syft._install_tool_windows') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.platform.system') + def test_get_components_windows(self, mock_platform, mock_install_windows, mock_run_syft, mock_get_list_component): + components = TestSyft._init_get_components(mock_platform, mock_install_windows, mock_run_syft, mock_get_list_component, "Windows", "./syft.exe") + + self.assertEqual(mock_install_windows.call_count, 1) + self.assertEqual(len(components), 2) + self.assertEqual(components[0].name, "component1") + self.assertEqual(components[1].name, "component2") + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.get_list_component') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.Syft._run_syft') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.Syft._install_tool_unix') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.platform.system') + def test_get_components_unsupported(self, mock_platform, mock_install_unix, mock_run_syft, mock_get_list_component): + components = TestSyft._init_get_components(mock_platform, mock_install_unix, mock_run_syft, mock_get_list_component, "Unsupported", None) + + self.assertEqual(mock_install_unix.call_count, 0) + self.assertIsNone(components) + + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.subprocess.run') + def test_run_syft(self, mock_subprocess_run): + mock_subprocess_run.return_value = MagicMock(returncode=0, stdout="output", stderr="") + + syft = Syft() + command_prefix = "./syft" + artifact = "artifact" + config = { + "SYFT": { + "OUTPUT_FORMAT": "json" + } + } + service_name = "test_service" + + result_file = syft._run_syft(command_prefix, artifact, config, service_name) + + self.assertEqual(result_file, "test_service_SBOM.json") + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.subprocess.run') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.logger') + def test_run_syft_failure(self, mock_logger ,mock_subprocess_run): + mock_subprocess_run.side_effect = [Exception("Error install"), MagicMock()] + + syft = Syft() + command_prefix = "./syft" + artifact = "artifact" + config = { + "SYFT": { + "OUTPUT_FORMAT": "json" + } + } + service_name = "test_service" + + result_file = syft._run_syft(command_prefix, artifact, config, service_name) + + self.assertIsNone(result_file) + mock_logger.error.assert_called_once_with("Error running syft: Error install") + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.subprocess.run') + def test_install_tool_unix_already_installed(self, mock_subprocess_run): + mock_subprocess_run.return_value = MagicMock(returncode=0, stdout="./syft\n".encode()) + + syft = Syft() + command_prefix = syft._install_tool_unix("syft.tar.gz", "http://example.com/syft.tar.gz", "syft") + + self.assertEqual(command_prefix, "./syft") + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.subprocess.run') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.tarfile.open') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.requests.get') + def test_install_tool_unix_success(self, mock_requests_get, mock_tarfile_open, mock_subprocess_run): + # Configurar los mocks + mock_subprocess_run.side_effect = [MagicMock(returncode=1), MagicMock()] + mock_requests_get.return_value.content = b"fake content" + mock_tarfile_open.return_value.__enter__.return_value.extract.return_value = None + + # Crear instancia de Syft + syft = Syft() + + # Llamar a la función + command_prefix = syft._install_tool_unix("syft.tar.gz", "http://example.com/syft.tar.gz", "syft") + + # Verificar que se llamaron las funciones esperadas + mock_requests_get.assert_called_once_with("http://example.com/syft.tar.gz", allow_redirects=True) + mock_tarfile_open.return_value.__enter__.return_value.extract.assert_called_once_with(member=mock_tarfile_open.return_value.__enter__.return_value.getmember("syft")) + self.assertEqual(command_prefix, "./syft") + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.subprocess.run') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.tarfile.open') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.requests.get') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.logger') + def test_install_tool_unix_failure(self, mock_logger, mock_requests_get, mock_tarfile_open, mock_subprocess_run): + # Configurar los mocks + mock_subprocess_run.side_effect = [MagicMock(returncode=1), Exception("Installation failed")] + mock_requests_get.return_value.content = b"fake content" + mock_tarfile_open.return_value.__enter__.return_value.extract.side_effect = Exception("Extraction failed") + + # Crear instancia de Syft + syft = Syft() + + # Llamar a la función y verificar que se lanza la excepción esperada + syft._install_tool_unix("syft.tar.gz", "http://example.com/syft.tar.gz", "syft") + + mock_logger.error.assert_called_once_with("Error installing syft: Extraction failed") + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.subprocess.run') + def test_install_tool_windows_already_installed(self, mock_subprocess_run): + # Configurar los mocks + mock_subprocess_run.return_value = MagicMock(returncode=0, stdout="C:\\syft.exe\n".encode()) + + # Crear instancia de Syft + syft = Syft() + + # Llamar a la función + command_prefix = syft._install_tool_windows("syft.zip", "http://example.com/syft.zip", "syft.exe") + + # Verificar que se llamaron las funciones esperadas + self.assertEqual(command_prefix, "C:\\syft.exe") + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.subprocess.run') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.requests.get') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.zipfile.ZipFile') + def test_install_tool_windows_success(self, mock_zipfile, mock_requests_get, mock_subprocess_run): + # Configurar los mocks + mock_subprocess_run.side_effect = [Exception("Not installed"), MagicMock()] + mock_requests_get.return_value.content = b"fake content" + mock_zipfile.return_value.__enter__.return_value.extract.return_value = None + + # Crear instancia de Syft + syft = Syft() + + # Llamar a la función + command_prefix = syft._install_tool_windows("syft.zip", "http://example.com/syft.zip", "syft.exe") + + # Verificar que se llamaron las funciones esperadas + mock_requests_get.assert_called_once_with("http://example.com/syft.zip", allow_redirects=True) + mock_zipfile.return_value.__enter__.return_value.extract.assert_called_once_with(member="syft.exe") + self.assertEqual(command_prefix, "./syft.exe") + + + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.subprocess.run') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.requests.get') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.zipfile.ZipFile') + @patch('devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.syft.syft.logger') + def test_install_tool_windows_failure(self, mock_logger ,mock_zipfile, mock_requests_get, mock_subprocess_run): + # Configurar los mocks + mock_subprocess_run.side_effect = [Exception("Not installed"), Exception("Installation failed")] + mock_requests_get.return_value.content = b"fake content" + mock_zipfile.return_value.__enter__.return_value.extract.side_effect = Exception("Extraction failed") + + # Crear instancia de Syft + syft = Syft() + + # Llamar a la función y verificar que se lanza la excepción esperada + syft._install_tool_windows("syft.zip", "http://example.com/syft.zip", "syft.exe") + + mock_logger.error.assert_called_once_with("Error installing syft: Extraction failed") + diff --git a/tools/devsecops_engine_tools/engine_core/test/infrastructure/entry_points/test_entry_point_core.py b/tools/devsecops_engine_tools/engine_core/test/infrastructure/entry_points/test_entry_point_core.py index fd6f19f20..f53c0c497 100644 --- a/tools/devsecops_engine_tools/engine_core/test/infrastructure/entry_points/test_entry_point_core.py +++ b/tools/devsecops_engine_tools/engine_core/test/infrastructure/entry_points/test_entry_point_core.py @@ -44,6 +44,7 @@ def test_init_engine_core( "remote_config_repo": "https://github.com/example/repo", "tool": "engine_iac", "send_metrics": "true", + "remote_config_branch": "" } # Call the function @@ -53,18 +54,20 @@ def test_init_engine_core( devops_platform_gateway=mock_devops_platform_gateway, print_table_gateway=mock.Mock(), metrics_manager_gateway=mock.Mock(), + sbom_tool_gateway=mock.Mock(), args=args, ) # Assert that the function calls were made with the expected arguments mock_devops_platform_gateway.get_remote_config.assert_called_once_with( - "https://github.com/example/repo", "/engine_core/ConfigTool.json" + "https://github.com/example/repo", "/engine_core/ConfigTool.json", "" ) mock_handle_scan.return_value.process.assert_called_once_with( { "remote_config_repo": "https://github.com/example/repo", "tool": "engine_iac", "send_metrics": "true", + "remote_config_branch": "" }, mock_config_tool, ) @@ -84,7 +87,7 @@ def test_init_engine_core_disabled(self, mock_print): mock_config_tool = { "BANNER": "DevSecOps Engine Tools", - "ENGINE_IAC": {"ENABLED": "false", "TOOL": "tool", "send_metrics": "false"} + "ENGINE_IAC": {"ENABLED": False, "TOOL": "tool"} } mock_devops_platform_gateway = mock.Mock() @@ -97,7 +100,8 @@ def test_init_engine_core_disabled(self, mock_print): devops_platform_gateway=mock_devops_platform_gateway, print_table_gateway=mock.Mock(), metrics_manager_gateway=mock.Mock(), - args={"remote_config_repo": "test", "tool": "engine_iac"}, + sbom_tool_gateway=mock.Mock(), + args={"remote_config_repo": "test", "tool": "engine_iac", "remote_config_branch": ""}, ) # Assert @@ -131,7 +135,8 @@ def test_init_engine_core_risk(self, mock_metrics, mock_handle_risk): devops_platform_gateway=mock_devops_platform_gateway, print_table_gateway=mock.Mock(), metrics_manager_gateway=mock.Mock(), - args={"remote_config_repo": "test", "tool": "engine_risk", "send_metrics": "true"}, + sbom_tool_gateway=mock.Mock(), + args={"remote_config_repo": "test", "tool": "engine_risk", "send_metrics": "true", "remote_config_branch": ""}, ) #Assert diff --git a/tools/devsecops_engine_tools/engine_dast/src/applications/runner_dast_scan.py b/tools/devsecops_engine_tools/engine_dast/src/applications/runner_dast_scan.py new file mode 100644 index 000000000..d15253f2b --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/applications/runner_dast_scan.py @@ -0,0 +1,103 @@ +import os +from typing import List +from devsecops_engine_tools.engine_dast.src.infrastructure.entry_points.entry_point_dast import ( + init_engine_dast, +) +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.nuclei.nuclei_tool import ( + NucleiTool, +) +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.jwt.jwt_object import ( + JwtObject, +) +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.jwt.jwt_tool import ( + JwtTool, +) +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.oauth.generic_oauth import ( + GenericOauth, +) +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.http.client.auth_client import ( + AuthClientCredential, +) +from devsecops_engine_tools.engine_dast.src.domain.model.api_config import ( + ApiConfig +) +from devsecops_engine_tools.engine_dast.src.domain.model.api_operation import ( + ApiOperation +) +from devsecops_engine_tools.engine_dast.src.domain.model.wa_config import ( + WaConfig +) +from devsecops_engine_tools.engine_dast.src.infrastructure.helpers.json_handler import ( + load_json_file +) + +def runner_engine_dast(dict_args, config_tool, secret_tool, devops_platform): + + if config_tool["TOOL"].lower() == "nuclei": # tool_gateway is the main Tool + tool_run = NucleiTool() + extra_tools = [] + target_config = None + + # Filling operations list with adapters + data = load_json_file(dict_args["dast_file_path"]) + + try: + + + if "operations" in data: # Api + operations: List = [] + for elem in data["operations"]: + security_type = elem["operation"]["security_auth"]["type"].lower() + if security_type == "jwt": + operations.append( + ApiOperation( + elem, + JwtObject( + elem["operation"]["security_auth"] + ))) + elif security_type == "oauth": + operations.append( + ApiOperation( + elem, + GenericOauth( + elem["operation"]["security_auth"], + data["endpoint"] + ) + ) + ) + else: + operations.append( + ApiOperation( + elem, + AuthClientCredential( + elem["operation"]["security_auth"] + ) + ) + ) + data["operations"] = operations + target_config = ApiConfig(data) + elif "WA" in data: # Web Application + if data["data"].get["security_auth"] == "oauth": + authentication_gateway = GenericOauth( + data["data"]["security_auth"] + ) + target_config = WaConfig(data, authentication_gateway) + else: + raise ValueError("Can't match if the target type is an api or a web application ") + + if any((k.lower() == "jwt") for k in config_tool["EXTRA_TOOLS"]) and \ + any(isinstance(operation.authentication_gateway, JwtObject) for operation in data["operations"] ): + extra_tools.append(JwtTool(target_config)) + + return init_engine_dast( + devops_platform_gateway=devops_platform, + tool_gateway=tool_run, + dict_args=dict_args, + secret_tool=secret_tool, + config_tool=config_tool, + extra_tools=extra_tools, + target_data=target_config + ) + except Exception as e: + raise Exception(f"Error engine_dast {e}") + \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/domain/model/api_config.py b/tools/devsecops_engine_tools/engine_dast/src/domain/model/api_config.py new file mode 100644 index 000000000..bea7983cc --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/domain/model/api_config.py @@ -0,0 +1,13 @@ +from typing import List +from devsecops_engine_tools.engine_dast.src.domain.model.api_operation import ApiOperation + + +class ApiConfig(): + def __init__(self, api_data: dict): + try: + self.target_type: str = "API" + self.endpoint: str = api_data["endpoint"] + self.rate_limit: str = api_data.get("rate_limit") + self.operations: "List[ApiOperation]" = api_data["operations"] + except KeyError: + raise KeyError("Missing configuration, validate the endpoint and every single operation") \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/domain/model/api_operation.py b/tools/devsecops_engine_tools/engine_dast/src/domain/model/api_operation.py new file mode 100644 index 000000000..e699582f4 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/domain/model/api_operation.py @@ -0,0 +1,13 @@ +from devsecops_engine_tools.engine_dast.src.domain.model.gateways.authentication_gateway import ( + AuthenticationGateway +) +class ApiOperation(): + def __init__(self, operation, authentication_gateway): + self.authentication_gateway: AuthenticationGateway = authentication_gateway + self.data: dict = operation + self.credentials = ("auth_header", "token") + + def authenticate(self): + self.credentials = self.authentication_gateway.get_credentials() + if self.credentials is not None: + self.data["operation"]["headers"][f'{self.credentials[0]}'] = f'{self.credentials[1]}' diff --git a/tools/devsecops_engine_tools/engine_dast/src/domain/model/config_tool.py b/tools/devsecops_engine_tools/engine_dast/src/domain/model/config_tool.py new file mode 100644 index 000000000..f53f3e3b3 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/domain/model/config_tool.py @@ -0,0 +1,16 @@ +from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold + + +class ConfigTool: + def __init__(self, json_data, tool): + self.version = json_data[tool].get("VERSION") + self.use_external_checks_dir = json_data[tool].get("USE_EXTERNAL_CHECKS_DIR") + self.external_dir_owner = json_data[tool].get("EXTERNAL_DIR_OWNER") + self.external_dir_repository = json_data[tool].get("EXTERNAL_DIR_REPOSITORY") + self.external_asset_name = json_data[tool].get("EXTERNAL_DIR_ASSET_NAME") + self.message_info_dast = json_data["MESSAGE_INFO_DAST"] + self.threshold = Threshold(json_data["THRESHOLD"]) + self.scope_pipeline = "" + self.exclusions = None + self.exclusions_all = None + self.exclusions_scope = None diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/.gitkeep b/tools/devsecops_engine_tools/engine_dast/src/domain/model/gateways/__init__.py similarity index 100% rename from tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/.gitkeep rename to tools/devsecops_engine_tools/engine_dast/src/domain/model/gateways/__init__.py diff --git a/tools/devsecops_engine_tools/engine_dast/src/domain/model/gateways/authentication_gateway.py b/tools/devsecops_engine_tools/engine_dast/src/domain/model/gateways/authentication_gateway.py new file mode 100644 index 000000000..3c9cbb32e --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/domain/model/gateways/authentication_gateway.py @@ -0,0 +1,7 @@ +from abc import ABCMeta, abstractmethod + + +class AuthenticationGateway(metaclass=ABCMeta): + @abstractmethod + def get_credentials(self) -> dict: + "get_credentials" diff --git a/tools/devsecops_engine_tools/engine_dast/src/domain/model/gateways/tool_gateway.py b/tools/devsecops_engine_tools/engine_dast/src/domain/model/gateways/tool_gateway.py new file mode 100644 index 000000000..01ab154d2 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/domain/model/gateways/tool_gateway.py @@ -0,0 +1,9 @@ +from abc import ABCMeta, abstractmethod + + +class ToolGateway(metaclass=ABCMeta): + @abstractmethod + def run_tool( + self, init_config_tool, exclusions, environment, pipeline, secret_tool + ) -> str: + "run_tool" diff --git a/tools/devsecops_engine_tools/engine_dast/src/domain/model/wa_config.py b/tools/devsecops_engine_tools/engine_dast/src/domain/model/wa_config.py new file mode 100644 index 000000000..da18e334f --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/domain/model/wa_config.py @@ -0,0 +1,10 @@ +class WaConfig: + def __init__(self, data: dict, authentication_gateway): + self.target_type: str = "WA" + self.url: str = data["endpoint"] + self.data: dict = data.wa_data + + def authenticate(self): + self.credentials = self.authentication_gateway.get_credentials() + if self.credentials is not None: + self.data["headers"][self.credentials[0]] = self.credentials[1] \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/domain/usecases/dast_scan.py b/tools/devsecops_engine_tools/engine_dast/src/domain/usecases/dast_scan.py new file mode 100644 index 000000000..f58eb795a --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/domain/usecases/dast_scan.py @@ -0,0 +1,127 @@ +from typing import ( + List, Tuple, Any +) +from devsecops_engine_tools.engine_dast.src.domain.model.gateways.tool_gateway import ( + ToolGateway, +) +from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import ( + DevopsPlatformGateway, +) +from devsecops_engine_tools.engine_core.src.domain.model.input_core import ( + InputCore, +) +from devsecops_engine_tools.engine_core.src.domain.model.exclusions import ( + Exclusions, +) +from devsecops_engine_tools.engine_dast.src.domain.model.config_tool import ( + ConfigTool, +) + +class DastScan: + def __init__( + self, + tool_gateway: ToolGateway, + devops_platform_gateway: DevopsPlatformGateway, + data_target, + aditional_tools: "List[ToolGateway]" + ): + self.tool_gateway = tool_gateway + self.devops_platform_gateway = devops_platform_gateway + self.data_target = data_target + self.other_tools = aditional_tools + + def complete_config_tool( + self, data_file_tool, exclusions, tool + ) -> "Tuple[ConfigTool, Any]": + config_tool = ConfigTool( + json_data=data_file_tool, + tool=tool, + ) + + config_tool.exclusions = exclusions + config_tool.scope_pipeline = self.devops_platform_gateway.get_variable( + "pipeline_name" + ) + + if config_tool.exclusions.get("All") is not None: + config_tool.exclusions_all = config_tool.exclusions.get("All").get( + tool + ) + if config_tool.exclusions.get(config_tool.scope_pipeline) is not None: + config_tool.exclusions_scope = config_tool.exclusions.get( + config_tool.scope_pipeline + ).get(config_tool) + + data_target_config = self.data_target + return config_tool, data_target_config + + def process( + self, dict_args, secret_tool, config_tool + ) -> "Tuple[List, InputCore]": + init_config_tool = self.devops_platform_gateway.get_remote_config( + dict_args["remote_config_repo"], "engine_dast/ConfigTool.json" + ) + + exclusions = self.devops_platform_gateway.get_remote_config( + dict_args["remote_config_repo"], + "engine_dast/Exclusions.json" + ) + + config_tool, data_target = self.complete_config_tool( + data_file_tool=init_config_tool, + exclusions=exclusions, + tool=config_tool["TOOL"], + ) + + finding_list, path_file_results = self.tool_gateway.run_tool( + target_data=data_target, + config_tool=config_tool, + secret_tool=secret_tool, + secret_external_checks=dict_args["token_external_checks"] + ) + #Here execute other tools and append to finding list + if len(self.other_tools) > 0: + for i in range(len(self.other_tools)): + extra_config_tool, data_target = self.complete_config_tool( + data_file_tool=init_config_tool, + exclusions=exclusions, + tool=self.other_tools[i].TOOL + ) + extra_finding_list = self.other_tools[i].run_tool( + target_data=data_target, + config_tool=extra_config_tool + ) + if len(extra_finding_list) > 0: + finding_list.extend(extra_finding_list) + + totalized_exclusions = [] + ( + totalized_exclusions.extend( + map( + lambda elem: Exclusions(**elem), config_tool.exclusions_all + ) + ) + if config_tool.exclusions_all is not None + else None + ) + ( + totalized_exclusions.extend( + map( + lambda elem: Exclusions(**elem), + config_tool.exclusions_scope, + ) + ) + if config_tool.exclusions_scope is not None + else None + ) + + input_core = InputCore( + totalized_exclusions=totalized_exclusions, + threshold_defined=config_tool.threshold, + path_file_results=path_file_results, + custom_message_break_build=config_tool.message_info_dast, + scope_pipeline=config_tool.scope_pipeline, + stage_pipeline=self.devops_platform_gateway.get_variable("stage"), + ) + + return finding_list, input_core \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/http/client/auth_client.py b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/http/client/auth_client.py new file mode 100644 index 000000000..a5b09632f --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/http/client/auth_client.py @@ -0,0 +1,12 @@ +from devsecops_engine_tools.engine_dast.src.domain.model.gateways.authentication_gateway import ( + AuthenticationGateway, +) + + +class AuthClientCredential(AuthenticationGateway): + def __init__(self, security_auth: dict): + self.client_id: str = security_auth.get("client_id") + self.client_secrets: str = security_auth.get("client_secret") + + def get_credentials(self): + return None \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/jwt/jwt_object.py b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/jwt/jwt_object.py new file mode 100644 index 000000000..19f250375 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/jwt/jwt_object.py @@ -0,0 +1,64 @@ +from time import ( + time +) +from secrets import ( + token_hex +) +from authlib.jose import ( + jwt +) +from devsecops_engine_tools.engine_dast.src.domain.model.gateways.authentication_gateway import ( + AuthenticationGateway, +) + + +class JwtObject(AuthenticationGateway): + def __init__(self, security_auth: dict): + self.type = "jwt" + self.private_key: str = security_auth.get("jwt_private_key") + self.algorithm: str = security_auth.get("jwt_algorithm") + self.iss: str = security_auth.get("jwt_iss") + self.sum: str = security_auth.get("jwt_sum") + self.aud: str = security_auth.get("jwt_aud") + self.iat: float = time() + self.exp: float = self.iat + 60 * 60 + self.nonce = token_hex(10) + self.payload: dict = {} + self.header: dict = {} + self.jwt_token: str = "" + self.header_name: str = security_auth.get("jwt_header_name") + self.init_header() + self.init_payload() + + def init_header(self) -> None: + self.header: dict = {"alg": self.algorithm} + + def init_payload(self) -> dict: + self.payload: dict = { + "iss": self.iss, + "sum": self.sum, + "aud": self.aud, + "exp": self.exp, + "iat": self.iat, + "nonce": self.nonce, + } + return self.payload + + def get_credentials(self) -> tuple: + """ + Generates JWT using a file with the configuration + + Returns: + + tuple: header and jwt + + """ + self.private_key = ( + self.private_key.replace(" ", "\n") + .replace("-----BEGIN\nPRIVATE\nKEY-----", "-----BEGIN PRIVATE KEY-----") + .replace("-----END\nPRIVATE\nKEY-----", "-----END PRIVATE KEY-----") + ) + self.jwt_token = jwt.encode(self.header, self.payload, self.private_key).decode( + "utf-8" + ) + return self.header_name, self.jwt_token \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/jwt/jwt_tool.py b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/jwt/jwt_tool.py new file mode 100644 index 000000000..a34e16bb7 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/jwt/jwt_tool.py @@ -0,0 +1,151 @@ +from typing import ( + List +) +from datetime import ( + datetime, +) +import jwt +from devsecops_engine_tools.engine_core.src.domain.model.finding import ( + Category, + Finding, +) +from devsecops_engine_tools.engine_dast.src.domain.model.gateways.tool_gateway import ( + ToolGateway, +) +from devsecops_engine_tools.engine_dast.src.domain.model.api_operation import ( + ApiOperation +) +from devsecops_engine_tools.engine_dast.src.infrastructure.helpers.file_generator_tool import ( + generate_file_from_tool, +) + +class JwtTool(ToolGateway): + + def __init__(self, target_config): + self.TOOL = "JWT" + self.BAD_JWT_ALG = ["none", "ES256", "ES384", "ES512"] + self.BAD_JWS_ALG = ["none", "ES256", "ES384", "ES512"] + self.GOOD_JWE_ALG = ["dir", "RSA-OAEP", "RSA-OAEP-256"] + self.GOOD_JWE_ENC = ["A256GCM"] + self.target_config = target_config + + def verify_jwt_alg(self, token): + "Evaluate JSON Web token's algorithm" + + map_id = "JWT_ALGORITHM" + alg = jwt.get_unverified_header(token)["alg"] + + if alg in self.BAD_JWT_ALG: #Is vulnerable + return { + "map_id": map_id, + "description": "msg" + } + + def verify_jws_alg(self, token): + """Evaluate JSON Web signature's algorithm""" + + map_id = "JWS_ALGORITHM" + alg = jwt.get_unverified_header(token)["alg"] + + if alg in self.BAD_JWS_ALG:#Is vulnerable + return { + "map_id": map_id, + "description": "msg" + } + + def verify_jwe(self, token): + """Evaluate JSON Web encryption's algorithm""" + + map_id = "JWE_ALGORITHM" + msg = "" + enc = jwt.get_unverified_header(token)["enc"] + alg = jwt.get_unverified_header(token)["alg"] + + if enc in self.GOOD_JWE_ENC: + if alg in self.GOOD_JWE_ALG:# Is not vulnerable + return + else: + msg = "Algorithm: " + alg + else: + msg = "Encryption: " + enc + + return { + "map_id": map_id, + "description": msg + } + + def check_token(self, token, jwt_details, config_tool): + "Validates JWT, JWS or JWE" + + hed = jwt.get_unverified_header(token) + + if "enc" in hed.keys(): + result = self.verify_jwe(token) + elif "typ" in hed.keys(): + result = self.verify_jwt_alg(token) + else: + result = self.verify_jws_alg(token) + + if result: + mapped_result = { + "check_id": config_tool["RULES"][result["map_id"]]["checkID"], + "cvss": config_tool["RULES"][result["map_id"]]["cvss"], + "matched-at": jwt_details["path"], + "description": result["msg"], + "severity": config_tool["RULES"][result["map_id"]]["severity"], + "remediation": result["remediation"] + } + return mapped_result + return None + def configure_tool(self, target_data): + """Method for group all operations that uses JWT""" + jwt_list: List[ApiOperation] = [] + for operation in target_data.operations: + if operation.authentication_gateway.type.lower() == "jwt": + jwt_list.append(operation) + return jwt_list + + def execute(self, jwt_config: List[ApiOperation], config_tool): + result_scans = [] + if len(jwt_config) > 0: + for jwt_operation in jwt_config: + result = self.check_token(token=jwt_operation.credentials[1], + jwt_details=jwt_operation.data["operation"], + config_tool=config_tool) + if result: + result_scans.append(result) + return result_scans + + def get_list_finding( + self, + result_scan_list: "List[dict]" + ) -> "List[Finding]": + list_open_findings = [] + if len(result_scan_list) > 0: + for scan in result_scan_list: + finding_open = Finding( + id=scan.get("check-id"), + cvss=scan.get("cvss"), + where=scan.get("matched-at"), + description=scan.get("description"), + severity=scan.get("severity").lower(), + identification_date=datetime.now().strftime("%d%m%Y"), + module="engine_dast", + category=Category("vulnerability"), + requirements=scan.get("remediation"), + tool="jwt", + published_date_cve=scan.get("published-cve") + ) + list_open_findings.append(finding_open) + return list_open_findings + + def run_tool(self, target_data, config_tool): + jwt_config = self.configure_tool(target_data) + result_scans = self.execute(jwt_config, config_tool) + if result_scans: + finding_list = self.get_list_finding(result_scans) + path_file_results = generate_file_from_tool( + self.TOOL, result_scans, config_tool + ) + return finding_list, path_file_results + return [] \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/helpers/.gitkeep b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/__init__.py similarity index 100% rename from tools/devsecops_engine_tools/engine_dast/src/infrastructure/helpers/.gitkeep rename to tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/__init__.py diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/nuclei_config.py b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/nuclei_config.py new file mode 100644 index 000000000..7fc4c7829 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/nuclei_config.py @@ -0,0 +1,70 @@ +from typing import List +import os +from ruamel.yaml import YAML +from json import dumps as json_dumps + +class NucleiConfig: + def __init__(self, target_config): + self.url: str = target_config.endpoint + self.target_type: str = target_config.target_type.lower() + self.custom_templates_dir: str = "" + self.output_file: str = "result_dast_scan.json" + self.yaml = YAML() + if self.target_type == "api": + self.data: List = target_config.operations + elif self.target_type == "wa": + self.data: dict = target_config.data + else: + raise ValueError("ERROR: The objective is not an api or web application type") + + def process_template_file( + self, + base_folder: str, + dest_folder: str, + template_name: str, + new_template_data: dict, + template_counter: int, + ) -> None: + new_template_name: str = "nuclei_template_" + str(template_counter) + ".yaml" + with open(template_name, "r") as template_file: # abrir archivo + template_data = self.yaml.load(template_file) + if "http" in template_data: + template_data["http"][0]["method"] = new_template_data["operation"]["method"] + template_data["http"][0]["path"] = [ + "{{BaseURL}}" + new_template_data["operation"]["path"] + ] + template_data["http"][0]["headers"] = new_template_data["operation"]["headers"] + if "payload" in new_template_data["operation"]: + body = json_dumps(new_template_data["operation"]["payload"]) + template_data["http"][0]["body"] = body + + new_template_path = os.path.join(dest_folder, new_template_name) + + with open(new_template_path, "w") as nf: + self.yaml.dump(template_data, nf) + + def process_templates_folder(self, base_folder: str) -> None: + if not os.path.exists(self.custom_templates_dir): + os.makedirs(self.custom_templates_dir) + + t_counter = 0 + for operation in self.data: + operation.authenticate() #Api Authentication + for root, dirs, files in os.walk(base_folder): + for file in files: + if file.endswith(".yaml"): + self.process_template_file( + base_folder=base_folder, + dest_folder=self.custom_templates_dir, + template_name=os.path.join(root, file), + new_template_data=operation.data, + template_counter=t_counter, + ) + t_counter += 1 + + def customize_templates(self, directory: str) -> None: + if self.target_type == "api": + self.custom_templates_dir = "customized-nuclei-templates" + self.process_templates_folder( + base_folder=directory + ) diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/nuclei_deserealizer.py b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/nuclei_deserealizer.py new file mode 100644 index 000000000..7bf8b49c6 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/nuclei_deserealizer.py @@ -0,0 +1,40 @@ +from dataclasses import ( + dataclass, +) +from datetime import ( + datetime, +) +from devsecops_engine_tools.engine_core.src.domain.model.finding import ( + Category, + Finding, +) + + +@dataclass +class NucleiDesealizator: + @classmethod + def get_list_finding( + cls, + results_scan_list: "list[dict]", + ) -> "list[Finding]": + list_open_findings = [] + + if len(results_scan_list) > 0: + for scan in results_scan_list: + finding_open = Finding( + id=scan.get("template-id"), + cvss=scan["info"].get("classification").get("cvss-score"), + where=scan.get("matched-at"), + description=scan["info"].get("description"), + severity=scan["info"].get("severity").lower(), + identification_date=datetime.now().strftime("%d%m%Y"), + module="engine_dast", + category=Category("vulnerability"), + requirements=scan["info"].get("remediation"), + tool="Nuclei", + published_date_cve=None + ) + list_open_findings.append(finding_open) + + return list_open_findings + diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/nuclei_tool.py b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/nuclei_tool.py new file mode 100644 index 000000000..926c8a2dc --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/nuclei/nuclei_tool.py @@ -0,0 +1,192 @@ +import os +import subprocess +import json +import platform +import requests +import shutil +from devsecops_engine_tools.engine_dast.src.domain.model.config_tool import ( + ConfigTool, +) +from devsecops_engine_tools.engine_dast.src.domain.model.gateways.tool_gateway import ( + ToolGateway, +) +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.nuclei.nuclei_config import ( + NucleiConfig, +) +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.nuclei.nuclei_deserealizer import ( + NucleiDesealizator, +) +from devsecops_engine_tools.engine_utilities.github.infrastructure.github_api import ( + GithubApi +) +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities import settings +from devsecops_engine_tools.engine_utilities.utils.utils import Utils + +logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() + + +class NucleiTool(ToolGateway): + + """A class that wraps the nuclei scanner functionality""" + + def __init__(self, target_config=None, data_config_cli=None): + """Initialize the class with the data from the config file and the cli""" + self.target_config = target_config + self.data_config_cli = data_config_cli + self.TOOL: str = "NUCLEI" + + def download_tool(self, version): + try: + base_url = f"https://github.com/projectdiscovery/nuclei/releases/download/v{version}/" + os_type = platform.system().lower() + + if os_type == "darwin": + file_name = f"nuclei_{version}_macOS_amd64.zip" + elif os_type == "linux": + file_name = f"nuclei_{version}_linux_amd64.zip" + elif os_type == "windows": + file_name = f"nuclei_{version}_windows_amd64.zip" + else: + logger.error(f"Error [101]: {os_type} is an unsupported OS type!") + return 101 # Unsupported OS type error + + url = f"{base_url}{file_name}" + + response = requests.get(url, allow_redirects=True) + if response.status_code != 200: + logger.error(f"Error [102]: Failed to download Nuclei version {version}. HTTP status code: {response.status_code}") + return 102 # Download failed error + + home_directory = os.path.expanduser("~") + zip_name = os.path.join(home_directory, file_name) + with open(zip_name, "wb") as f: + f.write(response.content) + + utils = Utils() + utils.unzip_file(zip_name, home_directory) + logger.info("Download and extraction completed successfully.") + return 0 # Success + except Exception as e: + logger.error(f"Error [103]: An exception occurred during download: {e}") + return 103 # General exception error + + def install_tool(self, version): + try: + nuclei_path = shutil.which("nuclei") + if nuclei_path: + logger.info(f"Success [200]: Nuclei is already installed at {nuclei_path}") + return 200 # Already installed + + logger.info("Nuclei not found. Downloading and installing...") + download_result = self.download_tool(version) + if download_result != 0: + logger.error(f"Error [104]: Download failed with error code {download_result}") + return 104 # Download step failed + + os_type = platform.system().lower() + home_directory = os.path.expanduser("~") + + if os_type == "darwin" or os_type == "linux": + executable_path = os.path.join(home_directory, "nuclei") + subprocess.run(["chmod", "+x", executable_path], check=True) + target_path = os.path.expanduser("~/.local/bin/nuclei") + shutil.move(executable_path, target_path) + logger.info(f"Success [201]: Nuclei installed at {target_path}") + return 201 # Installation successful + elif os_type == "windows": + executable_path = os.path.join(home_directory, "nuclei.exe") + target_path = os.path.join(home_directory, "AppData", "Local", "Programs", "nuclei.exe") + os.makedirs(os.path.dirname(target_path), exist_ok=True) + shutil.move(executable_path, target_path) + logger.info(f"Success [202]: Nuclei installed at {target_path}") + return {"status": 202, "path":target_path} + else: + logger.error(f"Error [105]: {os_type} is an unsupported OS type!") + return 105 # Unsupported OS type error + except subprocess.CalledProcessError as e: + logger.error(f"Error [106]: Command execution failed: {e}") + return 106 # Subprocess execution error + except Exception as e: + logger.error(f"Error [107]: An exception occurred during installation: {e}") + return 107 # General exception error + + + def configurate_external_checks( + self, config_tool: ConfigTool, secret, output_dir: str = "tmp" + ): + if secret is None: + logger.warning("The secret is not configured for external controls") + # Create configuration dir external checks + elif config_tool.use_external_checks_dir == "True": + github_api = GithubApi(secret["github_token"]) + github_api.download_latest_release_assets( + config_tool.external_dir_owner, + config_tool.external_dir_repository, + output_dir, + ) + return output_dir + config_tool.external_asset_name + else: + return None + + + def execute(self, command_prefix: str, target_config: NucleiConfig) -> dict: + """Interact with nuclei's core application""" + + command = ( + command_prefix + + " -duc " # disable automatic update check + + "-u " # target URLs/hosts to scan + + target_config.url + + (f" -ud {target_config.custom_templates_dir}" if target_config.custom_templates_dir else "") + + " -ni " # disable interactsh server + + "-dc " # disable clustering of requests + + "-tags " # Excute only templates with the especified tag + + target_config.target_type + + " -je " # file to export results in JSON format + + str(target_config.output_file) + + " -sr" + ) + + if command is not None: + result = subprocess.run( + command, + shell=True, + capture_output=True, + ) + error = result.stderr.decode().strip() if result.stderr else "" + if result.returncode != 0: + logger.warning( + f"Error executing nuclei: {error}") + with open(target_config.output_file, "r") as f: + json_response = json.load(f) + return json_response + + def run_tool(self, + target_data, + config_tool, + secret_tool, + secret_external_checks + ): + secret = None + if secret_tool is not None: + secret = secret_tool + elif secret_external_checks is not None: + secret = { + "github_token": ( + secret_external_checks.split("github")[1] + if "github" in secret_external_checks + else None + ) + } + result_install = self.install_tool(config_tool.version) + if result_install["status"] < 200: + return [], None + nuclei_config = NucleiConfig(target_data) + checks_directory = self.configurate_external_checks(config_tool, secret, "./tmp") #DATA PDN + if checks_directory: + nuclei_config.customize_templates(checks_directory) + result_scans = self.execute(result_install["path"],nuclei_config) + nuclei_deserealizator = NucleiDesealizator() + findings_list = nuclei_deserealizator.get_list_finding(result_scans) + return findings_list, nuclei_config.output_file diff --git a/tools/devsecops_engine_tools/engine_dast/test/.gitkeep b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/oauth/__init__.py similarity index 100% rename from tools/devsecops_engine_tools/engine_dast/test/.gitkeep rename to tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/oauth/__init__.py diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/oauth/generic_oauth.py b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/oauth/generic_oauth.py new file mode 100644 index 000000000..69f8de659 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/driven_adapters/oauth/generic_oauth.py @@ -0,0 +1,70 @@ +import requests +from devsecops_engine_tools.engine_dast.src.domain.model.gateways.authentication_gateway import ( + AuthenticationGateway +) +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities import settings + +logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() + + +class GenericOauth(AuthenticationGateway): + def __init__(self, data, endpoint): + self.data: dict = data + self.endpoint = endpoint + self.config = {} + + def process_data(self): + + self.config = { + "method": self.data["security_auth"].get("method", "POST"), + "path": self.data["security_auth"]["path"], + "grant_type": self.data["security_auth"]["grant_type"], + "scope": self.data["security_auth"].get("scope", None), + "headers": self.data["security_auth"]["headers"], + "client_secret": self.data["security_auth"]["client_secret"], + "client_id": self.data["security_auth"]["client_id"] + } + return self.config + + def get_access_token(self): + auth_config = self.process_data() + + if auth_config["grant_type"].lower() == "client_credentials": + return self.get_access_token_client_credentials() + else: + raise ValueError("OAuth: Grant type is not supported yet") + + def get_credentials(self): + return self.get_access_token() + + def get_access_token_client_credentials(self): + """Obtain access token using client credentials flow.""" + try: + required_keys = ["client_id", "client_secret"] + if not all(key in self.config for key in required_keys): + raise ValueError("One or more keys is missing in OAuth config") + + data = { + "client_id": self.config["client_id"], + "client_secret": self.config["client_secret"], + "grant_type": "client_credentials", + "scope": self.config["scope"] + } + + url = self.endpoint + self.config["path"] + headers = self.config["headers"] + response = requests.request( + self.config["method"], url, headers=headers, data=data, timeout=5 + ) + if 200 <= response.status_code < 300: + result = response.json()["access_token"] + return ("Authorization",f"Bearer {result}") + else: + print( + "OAuth: Can't obtain access token" + "token Unknown status " + "code {0}: -> {1}".format(response.status_code, response.text) + ) + except (ConnectionError, ValueError, KeyError) as e: + logger.warning("OAuth: Can't obtain access token: {0}".format(e)) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/entry_points/entry_point_dast.py b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/entry_points/entry_point_dast.py new file mode 100644 index 000000000..35115a45a --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/entry_points/entry_point_dast.py @@ -0,0 +1,16 @@ +from devsecops_engine_tools.engine_dast.src.domain.usecases.dast_scan import ( + DastScan, +) + + +def init_engine_dast( + devops_platform_gateway, + tool_gateway, + dict_args, + secret_tool, + config_tool, + extra_tools, + target_data +): + dast_scan = DastScan(tool_gateway, devops_platform_gateway, target_data, extra_tools) + return dast_scan.process(dict_args, secret_tool, config_tool) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/helpers/file_generator_tool.py b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/helpers/file_generator_tool.py new file mode 100644 index 000000000..41488e077 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/helpers/file_generator_tool.py @@ -0,0 +1,71 @@ +import json +import os + + +def generate_file_from_tool(tool, result_list, rules_doc): + if tool.lower() == "nuclei": + try: + result_one: dict = {} + result_two: dict = {} + if len(result_list) > 1: + result_one = result_list[0] + result_two = result_list[1] + file_name = "results.json" + results_data = { + "check_type": "Api and Web Application", + "results": { + "failed_checks": list( + map( + lambda x: update_field( + x, + "severity", + rules_doc[x.get("check_id")].get("severity").lower(), + ), + result_one.get("results", {}).get("failed_checks", []), + ) + ) + + list( + map( + lambda x: update_field( + x, + "severity", + rules_doc[x.get("check_id")].get("severity").lower(), + ), + result_two.get("results", {}).get("failed_checks", []), + ) + ) + }, + "summary": { + "passed": result_one.get("summary", {}).get("passed", 0) + + result_two.get("summary", {}).get("passed", 0), + "failed": result_one.get("summary", {}).get("failed", 0) + + result_two.get("summary", {}).get("failed", 0), + "skipped": result_one.get("summary", {}).get("skipped", 0) + + result_two.get("summary", {}).get("skipped", 0), + "parsing_errors": result_one.get("summary", {}).get( + "parsing_errors", 0 + ) + + result_one.get("summary", {}).get("parsing_errors", 0), + "resource_count": result_one.get("summary", {}).get( + "resource_count", 0 + ) + + result_two.get("summary", {}).get("resource_count", 0), + "nuclei_version": result_one.get("summary", {}).get( + "version", None + ), + }, + } + + with open(file_name, "w") as json_file: + json.dump(results_data, json_file, indent=4) + + absolute_path = os.path.abspath(file_name) + return absolute_path + except KeyError as e: + print(f"Dict KeyError in checks integration: {e}") + except Exception as ex: + print(f"Error during handling checkov json integrator {ex}") + + +def update_field(elem, field, new_value): + return {**elem, field: new_value} \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/src/infrastructure/helpers/json_handler.py b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/helpers/json_handler.py new file mode 100644 index 000000000..dcc6ff1d3 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/src/infrastructure/helpers/json_handler.py @@ -0,0 +1,13 @@ +import json + + +def load_json_file(file_path: str): + try: + with open(file_path, 'r') as file: + return json.load(file) + except FileNotFoundError: + raise FileNotFoundError(f"Error: The file '{file_path}' was not found.") + except json.JSONDecodeError: + raise json.JSONDecodeError(f"Error: The file '{file_path}' does not contain valid JSON.") + except IOError as e: + raise IOError(f"I/O Error: {e}") \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/test/.gitkeep b/tools/devsecops_engine_tools/engine_dast/test/applications/__init__.py similarity index 100% rename from tools/devsecops_engine_tools/engine_sca/engine_container/test/.gitkeep rename to tools/devsecops_engine_tools/engine_dast/test/applications/__init__.py diff --git a/tools/devsecops_engine_tools/engine_dast/test/applications/test_runner_dast_scan.py b/tools/devsecops_engine_tools/engine_dast/test/applications/test_runner_dast_scan.py new file mode 100644 index 000000000..a426a4739 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/test/applications/test_runner_dast_scan.py @@ -0,0 +1,147 @@ +import unittest +from unittest import mock +from devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan import ( + runner_engine_dast +) + +class TestRunnerEngineDast(unittest.TestCase): + DAST_FILE_PATH = "example_dast.json" + + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.load_json_file') + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.init_engine_dast') + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.NucleiTool') + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.JwtTool') + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.ApiConfig') + @mock.patch('os.environ', {'GITHUB_TOKEN': 'example_token'}) + def test_runner_engine_dast_with_jwt(self, mock_api_config,mock_jwt_tool, mock_nuclei_tool, + mock_init_engine_dast, mock_load_json_file): + # Configurar los valores de retorno de los mocks + mock_load_json_file.return_value = { + "endpoint": "https://example.com", + "operations": [ + { + "operation": { + "headers": {"accept": "/"}, + "method": "POST", + "path": "/example_path", + "security_auth": {"type": "jwt"} + } + } + ] + } + mock_nuclei_tool_instance = mock_nuclei_tool.return_value + mock_jwt_tool_instance = mock_jwt_tool.return_value + mock_init_engine_dast.return_value = (["finding1", "finding2"], "input_core_mock") + + # Mock de ApiConfig + mock_api_config_instance = mock_api_config.return_value + + # Configurar los argumentos + dict_args = { + "use_secrets_manager": "true", + "tool": "engine_dast", + "dast_file_path": TestRunnerEngineDast.DAST_FILE_PATH + } + config_tool = {"ENABLED": "true", "TOOL": "NUCLEI", "EXTRA_TOOLS": ["JWT"]} + secret_tool = {"github_token": "example_token"} + devops_platform_gateway = mock.Mock() + + # Llamar a la función + findings_list, input_core = runner_engine_dast(dict_args, config_tool, secret_tool, devops_platform_gateway) + + # Verificar que las funciones mockeadas fueron llamadas correctamente + mock_load_json_file.assert_called_once_with(dict_args["dast_file_path"]) + mock_init_engine_dast.assert_called_once_with( + devops_platform_gateway=devops_platform_gateway, + tool_gateway=mock_nuclei_tool_instance, + dict_args=dict_args, + secret_tool=secret_tool, + config_tool=config_tool, + extra_tools=[mock_jwt_tool_instance], + target_data=mock_api_config_instance # Verificar contra el mock de ApiConfig + ) + + # Verificar los resultados + self.assertEqual(findings_list, ["finding1", "finding2"]) + self.assertEqual(input_core, "input_core_mock") + + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.load_json_file') + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.init_engine_dast') + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.NucleiTool') + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.ApiConfig') + @mock.patch('os.environ', {'GITHUB_TOKEN': 'example_token'}) + def test_runner_engine_dast_with_oauth(self, + mock_api_config, + mock_nuclei_tool, + mock_init_engine_dast, + mock_load_json_file): + # Configurar los valores de retorno de los mocks + mock_load_json_file.return_value = { + "endpoint": "https://example.com", + "operations": [ + { + "operation": { + "headers": {"accept": "/"}, + "method": "POST", + "path": "/example_path", + "security_auth": {"type": "oauth"} + } + } + ] + } + mock_nuclei_tool_instance = mock_nuclei_tool.return_value + mock_init_engine_dast.return_value = (["finding1", "finding2"], "input_core_mock") + # Mock de ApiConfig + mock_api_config_instance = mock_api_config.return_value + + # Configurar los argumentos + dict_args = { + "use_secrets_manager": "true", + "tool": "engine_dast", + "dast_file_path": TestRunnerEngineDast.DAST_FILE_PATH + } + config_tool = {"ENABLED": "true", "TOOL": "NUCLEI", "EXTRA_TOOLS": []} + secret_tool = {"github_token": "example_token"} + devops_platform_gateway = mock.Mock() + + # Llamar a la función + findings_list, input_core = runner_engine_dast(dict_args, config_tool, secret_tool, devops_platform_gateway) + + # Verificar que las funciones mockeadas fueron llamadas correctamente + mock_load_json_file.assert_called_once_with(dict_args["dast_file_path"]) + mock_init_engine_dast.assert_called_once_with( + devops_platform_gateway=devops_platform_gateway, + tool_gateway=mock_nuclei_tool_instance, + dict_args=dict_args, + secret_tool=secret_tool, + config_tool=config_tool, + extra_tools=[], + target_data=mock_api_config_instance # Verificar contra el mock de ApiConfig + ) + + # Verificar los resultados + self.assertEqual(findings_list, ["finding1", "finding2"]) + self.assertEqual(input_core, "input_core_mock") + + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.load_json_file') + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.init_engine_dast') + @mock.patch('devsecops_engine_tools.engine_dast.src.applications.runner_dast_scan.NucleiTool') + def test_runner_engine_dast_with_invalid_target(self, mock_nuclei_tool, mock_init_engine_dast, mock_load_json_file): + # Configurar los valores de retorno de los mocks + mock_load_json_file.return_value = { + "invalid_key": "invalid_value" + } + + # Configurar los argumentos + dict_args = { + "use_secrets_manager": "true", + "tool": "engine_dast", + "dast_file_path": TestRunnerEngineDast.DAST_FILE_PATH + } + config_tool = {"ENABLED": "true", "TOOL": "NUCLEI", "EXTRA_TOOLS": []} + secret_tool = {"github_token": "example_token"} + devops_platform_gateway = mock.Mock() + + # Verifies exception + with self.assertRaises(Exception): + runner_engine_dast(dict_args, config_tool, secret_tool, devops_platform_gateway) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/test/domain/__init__.py b/tools/devsecops_engine_tools/engine_dast/test/domain/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_dast/test/domain/model/__init__.py b/tools/devsecops_engine_tools/engine_dast/test/domain/model/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_dast/test/domain/usecases/__init__.py b/tools/devsecops_engine_tools/engine_dast/test/domain/usecases/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_dast/test/domain/usecases/test_dast_scan.py b/tools/devsecops_engine_tools/engine_dast/test/domain/usecases/test_dast_scan.py new file mode 100644 index 000000000..12c1a4261 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/test/domain/usecases/test_dast_scan.py @@ -0,0 +1,80 @@ +import unittest +from unittest.mock import Mock, patch +from devsecops_engine_tools.engine_dast.src.domain.usecases.dast_scan import ( + DastScan, + ToolGateway, + DevopsPlatformGateway +) +class TestDastScan(unittest.TestCase): + + def setUp(self): + # Mocks + self.tool_gateway_mock = Mock(spec=ToolGateway) + self.tool_gateway_mock.TOOL = "jwt" + self.devops_platform_gateway_mock = Mock(spec=DevopsPlatformGateway) + self.data_target_mock = Mock() + self.additional_tools_mock = [self.tool_gateway_mock] + + # Instancia de DastScan + self.dast_scan = DastScan( + tool_gateway=self.tool_gateway_mock, + devops_platform_gateway=self.devops_platform_gateway_mock, + data_target=self.data_target_mock, + aditional_tools=self.additional_tools_mock + ) + + @patch('devsecops_engine_tools.engine_dast.src.domain.usecases.dast_scan.ConfigTool') + def test_complete_config_tool(self, config_tool_mock): + data_file_tool = {"key": "value"} + exclusions = {"All": {"tool_name": [{"type": "exclusion"}]}, + "pipeline_name": {"config_tool": [{"type": "exclusion_scope"}]}} + tool = "tool_name" + + config_tool_instance = config_tool_mock.return_value + config_tool_instance.exclusions = exclusions + self.devops_platform_gateway_mock.get_variable.return_value = "pipeline_name" + + config_tool, data_target_config = self.dast_scan.complete_config_tool(data_file_tool, exclusions, tool) + + config_tool_mock.assert_called_once_with(json_data=data_file_tool, tool=tool) + self.devops_platform_gateway_mock.get_variable.assert_called_once_with("pipeline_name") + self.assertEqual(config_tool, config_tool_instance) + self.assertEqual(data_target_config, self.data_target_mock) + + + @patch('devsecops_engine_tools.engine_dast.src.domain.usecases.dast_scan.ConfigTool') + @patch('devsecops_engine_tools.engine_dast.src.domain.usecases.dast_scan.Exclusions') + def test_process(self, excluions_mock, config_tool_mock): + dict_args = {"remote_config_repo": "some_repo", + "token_external_checks": "dummie_token"} + secret_tool = "some_token" + config_tool = {"TOOL": "tool_name"} + + init_config_tool = {"key": "init_value"} + exclusions = {"All": {"type": "exclusion"}, "pipeline_name": [{"type": "exclusion_scope"}]} + finding_list = ["finding1", "finding2"] + path_file_results = "path/to/results" + + self.devops_platform_gateway_mock.get_remote_config.side_effect = [init_config_tool, exclusions] + self.tool_gateway_mock.run_tool.return_value = (finding_list, path_file_results) + self.additional_tools_mock[0].run_tool.return_value = (finding_list, path_file_results) + + excluions_mock.side_effect = lambda **kwargs: kwargs + + result, _ = self.dast_scan.process(dict_args, secret_tool, config_tool) + + + self.devops_platform_gateway_mock.get_remote_config.assert_any_call( + dict_args["remote_config_repo"], "engine_dast/Exclusions.json" + ) + + self.tool_gateway_mock.run_tool.assert_called_with( + target_data=self.data_target_mock, + config_tool=config_tool_mock.return_value + ) + self.additional_tools_mock[0].run_tool.assert_called_with( + target_data=self.data_target_mock, + config_tool=config_tool_mock.return_value + ) + + self.assertEqual(result, finding_list) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/test/infrastructure/__init__.py b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/__init__.py b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/jwt/__init__.py b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/jwt/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/jwt/test_jwt_tool.py b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/jwt/test_jwt_tool.py new file mode 100644 index 000000000..097e29511 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/jwt/test_jwt_tool.py @@ -0,0 +1,128 @@ +import unittest +from unittest.mock import MagicMock, Mock, patch +from devsecops_engine_tools.engine_dast.src.domain.model import config_tool +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.jwt.jwt_tool import JwtTool + +class TestJwtTool(unittest.TestCase): + + def setUp(self): + self.target_config_mock = Mock() + self.jwt_tool = JwtTool(target_config=self.target_config_mock) + + @patch( + "devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.jwt.jwt_tool.jwt.get_unverified_header" + ) + def test_verify_jwt_alg(self, mock_get_unverified_header): + token = "dummy_token" + mock_get_unverified_header.return_value = {"alg": "none"} + result = self.jwt_tool.verify_jwt_alg(token) + + mock_get_unverified_header.assert_called_once_with(token) + self.assertEqual(result["map_id"], "JWT_ALGORITHM") + self.assertTrue("description" in result) + + @patch( + "devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.jwt.jwt_tool.jwt.get_unverified_header" + ) + def test_verify_jws_alg(self, mock_get_unverified_header): + token = "dummy_token" + mock_get_unverified_header.return_value = {"alg": "ES256"} + + result = self.jwt_tool.verify_jws_alg(token) + + mock_get_unverified_header.assert_called_once_with(token) + self.assertEqual(result["map_id"], "JWS_ALGORITHM") + self.assertTrue("description" in result) + + @patch( + "devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.jwt.jwt_tool.jwt.get_unverified_header" + ) + def test_verify_jwe(self, mock_get_unverified_header): + token = "dummy_token" + mock_get_unverified_header.side_effect = [ + {"enc": "A256GCM"}, + {"alg": "RSA-OAEP"} + ] + + result = self.jwt_tool.verify_jwe(token) + + self.assertEqual(mock_get_unverified_header.call_count, 2) + self.assertEqual(result, None) + + @patch( + "devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.jwt.jwt_tool.jwt.get_unverified_header" + ) + def test_check_token_jwe(self, mock_get_unverified_header): + token = "dummy_token" + jwt_details = {} + config_tool = MagicMock() + + mock_get_unverified_header.return_value = {"enc": "A256GCM"} + + with patch.object(self.jwt_tool, 'verify_jwe', return_value=None) as mock_verify_jwe: + result = self.jwt_tool.check_token(token, jwt_details, config_tool) + mock_verify_jwe.assert_called_once_with(token) + self.assertEqual(result, None) + + @patch( + "devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.jwt.jwt_tool.jwt.get_unverified_header" + ) + def test_check_token_jwt(self, mock_get_unverified_header): + token = "dummy_token" + jwt_details = {} + config_tool = MagicMock() + mock_get_unverified_header.return_value = {"typ": "JWT"} + + with patch.object(self.jwt_tool, 'verify_jwt_alg', return_value=None) as mock_verify_jwt_alg: + result = self.jwt_tool.check_token(token, jwt_details, config_tool) + mock_verify_jwt_alg.assert_called_once_with(token) + self.assertEqual(result, None) + + @patch( + "devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.jwt.jwt_tool.jwt.get_unverified_header" + ) + def test_check_token_jws(self, mock_get_unverified_header): + token = "dummy_token" + jwt_details = {} + config_tool = MagicMock() + mock_get_unverified_header.return_value = {} + + with patch.object(self.jwt_tool, 'verify_jws_alg', return_value=None) as mock_verify_jws_alg: + result = self.jwt_tool.check_token(token, jwt_details, config_tool) + mock_verify_jws_alg.assert_called_once_with(token) + self.assertEqual(result, None) + + def test_configure_tool(self): + operation_mock = Mock() + operation_mock.authentication_gateway.type = "JWT" + target_data_mock = Mock() + target_data_mock.operations = [operation_mock] + + result = self.jwt_tool.configure_tool(target_data_mock) + + self.assertIn(operation_mock, result) + + @patch( + "devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.jwt.jwt_tool.generate_file_from_tool" + ) + def test_run_tool(self, mock_generate_file_from_tool): + target_data_mock = Mock() + config_tool_mock = Mock() + jwt_operation_mock = Mock() + jwt_operation_mock.authenticate.return_value = "dummy_token" + self.jwt_tool.configure_tool = Mock(return_value=[jwt_operation_mock]) + self.jwt_tool.execute = Mock(return_value= + [{"check-id": "ENGINE_JWT_001", + "severity": "low", + "description": "weak alg"}]) + self.jwt_tool.deserialize_results = Mock(return_value=["finding"]) + + findings, _ = self.jwt_tool.run_tool(target_data_mock, config_tool_mock) + + self.jwt_tool.configure_tool.assert_called_once_with(target_data_mock) + self.jwt_tool.execute.assert_called_once_with([jwt_operation_mock], config_tool_mock) + mock_generate_file_from_tool.assert_called_once_with( + self.jwt_tool.TOOL, [{"check-id": "ENGINE_JWT_001", + "severity": "low", + "description": "weak alg"}], config_tool_mock + ) diff --git a/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/nuclei/__init__.py b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/nuclei/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/nuclei/test_nuclei_config.py b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/nuclei/test_nuclei_config.py new file mode 100644 index 000000000..e4ed6ab0a --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/nuclei/test_nuclei_config.py @@ -0,0 +1,83 @@ +import unittest +from unittest.mock import Mock, patch, mock_open +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.nuclei.nuclei_config import NucleiConfig + +class TestNucleiConfig(unittest.TestCase): + + def setUp(self): + self.target_config_api = Mock() + self.target_config_api.endpoint = "https://dummy.endpoint" + self.target_config_api.target_type = "api" + self.target_config_api.operations = [Mock(), Mock()] + + self.target_config_wa = Mock() + self.target_config_wa.endpoint = "https://dummy.endpoint" + self.target_config_wa.target_type = "wa" + self.target_config_wa.data = {"key": "value"} + + self.nuclei_api = NucleiConfig(self.target_config_api) + self.nuclei_wa = NucleiConfig(self.target_config_wa) + + def test_init_api(self): + self.assertEqual(self.nuclei_api.url, "https://dummy.endpoint") + self.assertEqual(self.nuclei_api.target_type, "api") + self.assertEqual(self.nuclei_api.data, self.target_config_api.operations) + + def test_init_wa(self): + self.assertEqual(self.nuclei_wa.url, "https://dummy.endpoint") + self.assertEqual(self.nuclei_wa.target_type, "wa") + self.assertEqual(self.nuclei_wa.data, self.target_config_wa.data) + + def test_init_invalid_target_type(self): + target_config_invalid = Mock() + target_config_invalid.target_type = "invalid" + with self.assertRaises(ValueError): + NucleiConfig(target_config_invalid) + + @patch('os.makedirs') + @patch('os.path.exists', return_value=False) + def test_process_templates_folder(self, mock_exists, mock_makedirs): + base_folder = "dummy_base_folder" + self.nuclei_api.custom_templates_dir = "dummy_custom_templates_dir" + + with patch('os.walk', return_value=[('root', [], ['file.yaml'])]), \ + patch('builtins.open', mock_open(read_data="https: {}")), \ + patch('devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.nuclei.nuclei_config.NucleiConfig.process_template_file') as mock_process_template_file: + + self.nuclei_api.process_templates_folder(base_folder) + mock_exists.assert_called_once_with(self.nuclei_api.custom_templates_dir) + mock_makedirs.assert_called_once_with(self.nuclei_api.custom_templates_dir) + + @patch('builtins.open', new_callable=mock_open, read_data="https: {}") + @patch('devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.nuclei.nuclei_config.YAML.load', + return_value={"https": [{}]}) + @patch('devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.nuclei.nuclei_config.YAML.dump') + def test_process_template_file(self, mock_dump, mock_load, mock_open): + base_folder = "dummy_base_folder" + dest_folder = "dummy_dest_folder" + template_name = "dummy_template.yaml" + new_template_data = { + "operation": { + "method": "GET", + "path": "/dummy_path", + "headers": {"Content-Type": "application/json"}, + "payload": {"key": "value"} + } + } + template_counter = 0 + + self.nuclei_api.process_template_file(base_folder, + dest_folder, + template_name, + new_template_data, + template_counter) + + mock_load.assert_called_once() + mock_dump.assert_called_once() + + @patch('devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.nuclei.nuclei_config.NucleiConfig.process_templates_folder') + def test_customize_templates(self, mock_process_templates_folder): + directory = "dummy_directory" + self.nuclei_api.customize_templates(directory) + self.assertEqual(self.nuclei_api.custom_templates_dir, "customized-nuclei-templates") + mock_process_templates_folder.assert_any_call(base_folder=directory) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/nuclei/test_nuclei_tool.py b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/nuclei/test_nuclei_tool.py new file mode 100644 index 000000000..c79383fda --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/nuclei/test_nuclei_tool.py @@ -0,0 +1,43 @@ +import unittest +from unittest.mock import Mock, patch, mock_open +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.nuclei.nuclei_tool import ( + NucleiTool, + NucleiConfig + ) + +class TestNucleiTool(unittest.TestCase): + + def setUp(self): + self.target_config = Mock() + self.target_config.endpoint = "https://dummy.endpoint" + self.target_config.target_type = "api" + self.target_config.custom_templates_dir = "dummy_templates_dir" + self.target_config.output_file = "dummy_output_file.json" + + self.config_tool = { + "NUCLEI": { + "VERSION": "2.3.296", + "USE_EXTERNAL_CHECKS_GIT": "False", + "EXTERNAL_CHECKS_GIT": "git@github.com:example/Checks.git//rules", + "USE_EXTERNAL_CHECKS_DIR": "True", + "EXTERNAL_DIR_OWNER": "username", + "EXTERNAL_DIR_REPOSITORY": "engine-dast-nuclei-templates", + "EXTERNAL_DIR_ASSET_NAME": "rules/engine_dast/nuclei", + "EXCLUSIONS_PATH": "/engine_dast/Exclusions.json", + "EXTERNAL_CHECKS_PATH": "/nuclei-templates", + "MESSAGE_INFO_DAST": "If you have doubts, visit https://example.com/t/" + } + } + self.token = "dummy_token" + + self.nuclei_tool = NucleiTool(target_config=self.target_config) + + @patch('os.environ.get', return_value="true") + @patch('builtins.open', new_callable=mock_open, read_data='{"key": "value"}') + @patch('json.load', return_value={"key": "value"}) + def test_execute(self, mock_json_load, mock_open, mock_os_environ): + target_config = NucleiConfig(self.target_config) + result = self.nuclei_tool.execute("", target_config) + mock_open.assert_called_once_with(target_config.output_file, 'r') + mock_json_load.assert_called_once() + self.assertEqual(result, {"key": "value"}) diff --git a/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/oauth/__init__.py b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/oauth/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/oauth/test_generic_oauth.py b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/oauth/test_generic_oauth.py new file mode 100644 index 000000000..ff329a67d --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/driven_adapters/oauth/test_generic_oauth.py @@ -0,0 +1,56 @@ +import unittest +from unittest.mock import Mock, patch +from devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.oauth.generic_oauth import GenericOauth + +class TestGenericOauth(unittest.TestCase): + + def setUp(self): + self.data = { + "security_auth": { + "type": "oauth", + "method": "POST", + "path": "oauth2/token", + "grant_type": "client_credentials", + "scope": "TermExample:read:user", + "client_id": "dummy-id", + "client_secret": "dummy-secret", + "headers": { + "content-type": "application/x-www-form-urlencoded", + "accept": "application/json" + } + } + } + self.oauth = GenericOauth(self.data, "example.com") + + def test_process_data(self): + config = self.oauth.process_data() + + expected_config = { + "method": "POST", + "path": "oauth2/token", + "grant_type": "client_credentials", + "scope": "TermExample:read:user", + "headers": { + "content-type": "application/x-www-form-urlencoded", + "accept": "application/json" + }, + "client_secret": "dummy-secret", + "client_id": "dummy-id" + } + self.assertEqual(config, expected_config) + + @patch('devsecops_engine_tools.engine_dast.src.infrastructure.driven_adapters.oauth.generic_oauth.requests.request') + def test_get_access_token_client_credentials_flow(self, mock_request): + self.oauth.config = self.oauth.process_data() + self.oauth.config["tenant_id"] = "dummy_tenant_id" + response_mock = Mock() + response_mock.status_code = 200 + response_mock.json.return_value = {"access_token": "dummy_access_token"} + mock_request.return_value = response_mock + + token = self.oauth.get_access_token_client_credentials() + + mock_request.assert_called_once_with( + 'POST', 'example.comoauth2/token', headers={'content-type': 'application/x-www-form-urlencoded', 'accept': 'application/json'}, data={'client_id': 'dummy-id', 'client_secret': 'dummy-secret', 'grant_type': 'client_credentials', 'scope': 'TermExample:read:user'}, timeout=5 + ) + self.assertEqual(token, ('Authorization', 'Bearer dummy_access_token')) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_dast/test/infrastructure/helpers/test_dast_file_generator_tool.py b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/helpers/test_dast_file_generator_tool.py new file mode 100644 index 000000000..1cfe9a5ab --- /dev/null +++ b/tools/devsecops_engine_tools/engine_dast/test/infrastructure/helpers/test_dast_file_generator_tool.py @@ -0,0 +1,67 @@ +import unittest +from unittest.mock import patch, mock_open, call +import json + +# Importa la función a probar +from devsecops_engine_tools.engine_dast.src.infrastructure.helpers.file_generator_tool import generate_file_from_tool, update_field + +class TestGenerateFileFromTool(unittest.TestCase): + @patch("devsecops_engine_tools.engine_dast.src.infrastructure.helpers.file_generator_tool.open", new_callable=mock_open) # Simula 'open' + @patch("devsecops_engine_tools.engine_dast.src.infrastructure.helpers.file_generator_tool.os.path.abspath") # Simula 'os.path.abspath' + def test_generate_file_from_tool_nuclei(self, mock_abspath, mock_open): + # Datos de entrada simulados + tool = "nuclei" + result_list = [ + { + "results": { + "failed_checks": [ + {"check_id": "id1", "severity": "high"}, + {"check_id": "id2", "severity": "medium"}, + ] + }, + "summary": { + "passed": 5, + "failed": 2, + "skipped": 1, + "parsing_errors": 0, + "resource_count": 10, + "version": "2.4.1", + } + }, + { + "results": { + "failed_checks": [ + {"check_id": "id3", "severity": "low"}, + ] + }, + "summary": { + "passed": 2, + "failed": 1, + "skipped": 0, + "parsing_errors": 0, + "resource_count": 5, + "version": "2.4.1", + } + } + ] + rules_doc = { + "id1": {"severity": "critical"}, + "id2": {"severity": "high"}, + "id3": {"severity": "low"}, + } + + # Valores de retorno simulados + mock_abspath.return_value = "/mocked/path/results.json" + + # Llamada a la función + result = generate_file_from_tool(tool, result_list, rules_doc) + + # Verificación del nombre de archivo devuelto + self.assertEqual(result, "/mocked/path/results.json") + + # Verifica que 'open' se llame con el nombre de archivo correcto + mock_open.assert_called_once_with("results.json", "w") + + # Obtener la instancia del archivo simulado + handle = mock_open() + handle.write.assert_called() # Verifica que write se haya llamado diff --git a/tools/devsecops_engine_tools/engine_risk/src/applications/runner_engine_risk.py b/tools/devsecops_engine_tools/engine_risk/src/applications/runner_engine_risk.py index d868dcb29..31af0ca43 100644 --- a/tools/devsecops_engine_tools/engine_risk/src/applications/runner_engine_risk.py +++ b/tools/devsecops_engine_tools/engine_risk/src/applications/runner_engine_risk.py @@ -13,7 +13,12 @@ def runner_engine_risk( - dict_args, findings, vm_exclusions, devops_platform_gateway, print_table_gateway + dict_args, + findings, + vm_exclusions, + services, + devops_platform_gateway, + print_table_gateway, ): add_epss_gateway = FirstCsv() @@ -23,5 +28,6 @@ def runner_engine_risk( print_table_gateway, dict_args, findings, + services, vm_exclusions, ) diff --git a/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/break_build.py b/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/break_build.py index 9247acf2e..a1082375b 100644 --- a/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/break_build.py +++ b/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/break_build.py @@ -25,6 +25,7 @@ def __init__( vm_exclusions: "list[Exclusions]", report_list: "list[Report]", all_report: "list[Report]", + threshold: any, ): self.devops_platform_gateway = devops_platform_gateway self.printer_table_gateway = printer_table_gateway @@ -33,6 +34,7 @@ def __init__( self.vm_exclusions = vm_exclusions self.report_list = report_list self.all_report = all_report + self.threshold = threshold self.break_build = False self.warning_build = False self.report_breaker = [] @@ -117,25 +119,26 @@ def _breaker(self): print(self.devops_platform_gateway.result_pipeline("succeeded")) def _remediation_rate_control(self, all_report: "list[Report]"): - remote_config = self.remote_config - remediation_rate_value = self._get_percentage( - (sum(1 for report in all_report if report.mitigated)) / len(all_report) - ) - risk_threshold = remote_config["THRESHOLD"]["REMEDIATION_RATE"] + mitigated = sum(1 for report in all_report if report.mitigated) + total = len(all_report) + print(f"Mitigated count: {mitigated} Total count: {total}") + remediation_rate_value = self._get_percentage(mitigated / total) + + risk_threshold = self._get_remediation_rate_threshold(total) self.remediation_rate = remediation_rate_value if remediation_rate_value >= (risk_threshold + 5): print( self.devops_platform_gateway.message( "succeeded", - f"Remediation Rate {remediation_rate_value}% is greater than {risk_threshold}%", + f"Remediation rate {remediation_rate_value}% is greater than {risk_threshold}%", ) ) elif remediation_rate_value >= risk_threshold: print( self.devops_platform_gateway.message( "warning", - f"Remediation Rate {remediation_rate_value}% is close to {risk_threshold}%", + f"Remediation rate {remediation_rate_value}% is close to {risk_threshold}%", ) ) self.warning_build = True @@ -143,22 +146,21 @@ def _remediation_rate_control(self, all_report: "list[Report]"): print( self.devops_platform_gateway.message( "error", - f"Remediation Rate {remediation_rate_value}% is less than {risk_threshold}%", + f"Remediation rate {remediation_rate_value}% is less than {risk_threshold}%", ) ) self.break_build = True + def _get_remediation_rate_threshold(self, total): + remediation_rate = self.threshold["REMEDIATION_RATE"] + for key in sorted(remediation_rate.keys(), key=lambda x: int(x) if x.isdigit() else float('inf')): + if key.isdigit() and total <= int(key): + return remediation_rate[key] + return remediation_rate["other"] + def _get_percentage(self, decimal): return round(decimal * 100, 3) - def _get_applied_exclusion(self, report: Report): - for exclusion in self.exclusions: - if exclusion.id and (report.id == exclusion.id): - return exclusion - elif exclusion.id and (report.vuln_id_from_tool == exclusion.id): - return exclusion - return None - def _map_applied_exclusion(self, exclusions: "list[Exclusions]"): return [ { @@ -168,33 +170,55 @@ def _map_applied_exclusion(self, exclusions: "list[Exclusions]"): "create_date": exclusion.create_date, "expired_date": exclusion.expired_date, "reason": exclusion.reason, + "vm_id": exclusion.vm_id, + "vm_id_url": exclusion.vm_id_url, + "service": exclusion.service, + "tags": exclusion.tags, } for exclusion in exclusions ] def _apply_exclusions(self, report_list: "list[Report]"): - new_report_list = [] + filtered_reports = [] applied_exclusions = [] - exclusions_ids = {exclusion.id for exclusion in self.exclusions if exclusion.id} for report in report_list: - if report.vuln_id_from_tool and ( - report.vuln_id_from_tool in exclusions_ids - ): - applied_exclusions.append(self._get_applied_exclusion(report)) - elif report.id and (report.id in exclusions_ids): - applied_exclusions.append(self._get_applied_exclusion(report)) - else: + exclude = False + for exclusion in self.exclusions: + if ( + ( + report.vuln_id_from_tool + and report.vuln_id_from_tool == exclusion.id + ) + or (report.id and report.id == exclusion.id) + or (report.vm_id and exclusion.id in report.vm_id) + ) and ((exclusion.where in report.where) or (exclusion.where == "all")): + if not exclusion.check_in_desc: + exclude = True + else: + for item in exclusion.check_in_desc: + if item in report.vul_description: + exclude = True + break + if exclude: + exclusion_copy = copy.deepcopy(exclusion) + exclusion_copy.vm_id = report.vm_id + exclusion_copy.vm_id_url = report.vm_id_url + exclusion_copy.service = report.service + exclusion_copy.tags = report.tags + applied_exclusions.append(exclusion_copy) + break + if not exclude: report.reason = "Remediation Rate" - new_report_list.append(report) + filtered_reports.append(report) - return new_report_list, applied_exclusions + return filtered_reports, applied_exclusions def _tag_blacklist_control(self, report_list: "list[Report]"): remote_config = self.remote_config if report_list: - tag_blacklist = set(remote_config["THRESHOLD"]["TAG_BLACKLIST"]) - tag_age_threshold = remote_config["THRESHOLD"]["TAG_MAX_AGE"] + tag_blacklist = set(remote_config["TAG_BLACKLIST"]) + tag_age_threshold = self.threshold["TAG_MAX_AGE"] filtered_reports_above_threshold = [ (report, tag) @@ -215,7 +239,7 @@ def _tag_blacklist_control(self, report_list: "list[Report]"): print( self.devops_platform_gateway.message( "error", - f"Report {report.vuln_id_from_tool if report.vuln_id_from_tool else report.id} with tag {tag} is blacklisted and age {report.age} is above threshold {tag_age_threshold}", + f"Report {report.vm_id} with tag {tag} is blacklisted and age {report.age} is above threshold {tag_age_threshold}", ) ) @@ -223,7 +247,7 @@ def _tag_blacklist_control(self, report_list: "list[Report]"): print( self.devops_platform_gateway.message( "warning", - f"Report {report.vuln_id_from_tool if report.vuln_id_from_tool else report.id} with tag {tag} is blacklisted but age {report.age} is below threshold {tag_age_threshold}", + f"Report {report.vm_id} with tag {tag} is blacklisted but age {report.age} is below threshold {tag_age_threshold}", ) ) @@ -238,14 +262,17 @@ def _tag_blacklist_control(self, report_list: "list[Report]"): def _risk_score_control(self, report_list: "list[Report]"): remote_config = self.remote_config - risk_score_threshold = remote_config["THRESHOLD"]["RISK_SCORE"] + risk_score_threshold = self.threshold["RISK_SCORE"] break_build = False if report_list: for report in report_list: report.risk_score = round( remote_config["WEIGHTS"]["severity"].get(report.severity.lower(), 0) + remote_config["WEIGHTS"]["epss_score"] * report.epss_score - + remote_config["WEIGHTS"]["age"] * report.age + + min( + remote_config["WEIGHTS"]["age"] * report.age, + remote_config["WEIGHTS"]["max_age"], + ) + sum( remote_config["WEIGHTS"]["tags"].get(tag, 0) for tag in report.tags @@ -256,9 +283,7 @@ def _risk_score_control(self, report_list: "list[Report]"): break_build = True report.reason = "Risk Score" self.report_breaker.append(copy.deepcopy(report)) - print( - "Below are open vulnerabilities from Vulnerability Management Platform" - ) + print("Below are open findings from Vulnerability Management Platform") self.printer_table_gateway.print_table_report( report_list, ) @@ -282,7 +307,8 @@ def _risk_score_control(self, report_list: "list[Report]"): else: print( self.devops_platform_gateway.message( - "succeeded", "There are no vulnerabilities" + "succeeded", + "There are no open findings from Vulnerability Management Platform", ) ) @@ -293,7 +319,7 @@ def _print_exclusions(self, applied_exclusions: "list[Exclusions]"): "warning", "Bellow are all findings that were excepted" ) ) - self.printer_table_gateway.print_table_exclusions(applied_exclusions) + self.printer_table_gateway.print_table_report_exlusions(applied_exclusions) for reason, total in Counter( map(lambda x: x["reason"], applied_exclusions) ).items(): diff --git a/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/check_threshold.py b/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/check_threshold.py new file mode 100644 index 000000000..7877c7be2 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/check_threshold.py @@ -0,0 +1,19 @@ +import re + + +class CheckThreshold: + def __init__(self, pipeline_name, threshold, risk_exclusions): + self.pipeline_name = pipeline_name + self.threshold = threshold + self.risk_exclusions = risk_exclusions + + def process(self): + if (self.pipeline_name in self.risk_exclusions.keys()) and ( + self.risk_exclusions[self.pipeline_name].get("THRESHOLD", None) + ): + return self.risk_exclusions[self.pipeline_name]["THRESHOLD"] + elif "BY_PATTERN_SEARCH" in self.risk_exclusions.keys(): + for pattern, values in self.risk_exclusions["BY_PATTERN_SEARCH"].items(): + if re.match(pattern, self.pipeline_name): + return values["THRESHOLD"] + return self.threshold diff --git a/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/get_exclusions.py b/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/get_exclusions.py index a2a7c47e3..90adf26c2 100644 --- a/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/get_exclusions.py +++ b/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/get_exclusions.py @@ -11,18 +11,18 @@ def __init__( findings, risk_config, risk_exclusions, - pipeline_name, + services, ): self.devops_platform_gateway = devops_platform_gateway self.dict_args = dict_args self.findings = findings self.risk_config = risk_config self.risk_exclusions = risk_exclusions - self.pipeline_name = pipeline_name + self.services = services def process(self): core_config = self.devops_platform_gateway.get_remote_config( - self.dict_args["remote_config_repo"], "engine_core/ConfigTool.json" + self.dict_args["remote_config_repo"], "engine_core/ConfigTool.json", self.dict_args["remote_config_branch"] ) unique_tags = self._get_unique_tags() exclusions = [] @@ -42,14 +42,15 @@ def _get_risk_exclusions(self): def _get_exclusions_by_practice(self, core_config, practice, path): exclusions_config = self.devops_platform_gateway.get_remote_config( - self.dict_args["remote_config_repo"], path + self.dict_args["remote_config_repo"], path, self.dict_args["remote_config_branch"] ) tool = core_config[practice.upper()]["TOOL"] return self._get_exclusions(exclusions_config, tool) def _get_exclusions(self, config, key): exclusions = [] - for scope in ["All", self.pipeline_name]: + scope_list = ["All"] + self.services + for scope in scope_list: if config.get(scope, None) and config[scope].get(key, None): exclusions.extend( [ @@ -57,6 +58,7 @@ def _get_exclusions(self, config, key): **exclusion, ) for exclusion in config[scope][key] + if exclusion.get("id", None) ] ) return exclusions diff --git a/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/handle_filters.py b/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/handle_filters.py index 8a4f1bf46..55dd6d637 100644 --- a/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/handle_filters.py +++ b/tools/devsecops_engine_tools/engine_risk/src/domain/usecases/handle_filters.py @@ -1,9 +1,72 @@ +import copy + + class HandleFilters: def filter(self, findings): active_findings = self._get_active_findings(findings) self._get_priority_vulnerability(active_findings) return active_findings + def filter_duplicated(self, findings): + unique_findings = [] + findings_map = {} + + for finding in findings: + key = (finding.where, tuple(finding.id), finding.vuln_id_from_tool) + if key in findings_map: + existing_finding = findings_map[key] + combined_services = existing_finding.service.split() + [ + s + for s in finding.service.split() + if s not in existing_finding.service.split() + ] + combined_vm_ids = existing_finding.vm_id.split() + [ + vm + for vm in finding.vm_id.split() + if vm not in existing_finding.vm_id.split() + ] + combined_vm_id_urls = existing_finding.vm_id_url.split() + [ + vm_url + for vm_url in finding.vm_id_url.split() + if vm_url not in existing_finding.vm_id_url.split() + ] + if finding.age >= existing_finding.age: + new_finding = copy.deepcopy(finding) + new_finding.service = " ".join(combined_services) + new_finding.vm_id = " ".join(combined_vm_ids) + new_finding.vm_id_url = " ".join(combined_vm_id_urls) + findings_map[key] = new_finding + else: + existing_finding.service = " ".join(combined_services) + existing_finding.vm_id = " ".join(combined_vm_ids) + existing_finding.vm_id_url = " ".join(combined_vm_id_urls) + else: + findings_map[key] = copy.deepcopy(finding) + + unique_findings = list(findings_map.values()) + return unique_findings + + def filter_tags_days(self, devops_platform_gateway, remote_config, findings): + tag_exclusion_days = remote_config["TAG_EXCLUSION_DAYS"] + filtered_findings = [] + + for finding in findings: + exclude = False + for tag in finding.tags: + if tag in tag_exclusion_days and finding.age < tag_exclusion_days[tag]: + exclude = True + print( + devops_platform_gateway.message( + "warning", + f"Report {finding.vm_id} with tag '{tag}' and age {finding.age} days is being excluded. It will be considered in {tag_exclusion_days[tag] - finding.age} days.", + ) + ) + break + if not exclude: + filtered_findings.append(finding) + + return filtered_findings + def _get_active_findings(self, findings): return list( filter( diff --git a/tools/devsecops_engine_tools/engine_risk/src/infrastructure/entry_points/entry_point_risk.py b/tools/devsecops_engine_tools/engine_risk/src/infrastructure/entry_points/entry_point_risk.py index d43516131..4529835f0 100644 --- a/tools/devsecops_engine_tools/engine_risk/src/infrastructure/entry_points/entry_point_risk.py +++ b/tools/devsecops_engine_tools/engine_risk/src/infrastructure/entry_points/entry_point_risk.py @@ -10,10 +10,11 @@ from devsecops_engine_tools.engine_risk.src.domain.usecases.get_exclusions import ( GetExclusions, ) +from devsecops_engine_tools.engine_risk.src.domain.usecases.check_threshold import ( + CheckThreshold, +) -import re - from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings @@ -26,51 +27,17 @@ def init_engine_risk( print_table_gateway, dict_args, findings, + services, vm_exclusions, ): remote_config = devops_platform_gateway.get_remote_config( - dict_args["remote_config_repo"], "engine_risk/ConfigTool.json" + dict_args["remote_config_repo"], "engine_risk/ConfigTool.json", dict_args["remote_config_branch"] ) risk_exclusions = devops_platform_gateway.get_remote_config( - dict_args["remote_config_repo"], "engine_risk/Exclusions.json" + dict_args["remote_config_repo"], "engine_risk/Exclusions.json", dict_args["remote_config_branch"] ) pipeline_name = devops_platform_gateway.get_variable("pipeline_name") - if should_skip_analysis(remote_config, pipeline_name, risk_exclusions): - print("Tool skipped by DevSecOps Policy.") - logger.info("Tool skipped by DevSecOps Policy.") - return - return process_findings( - findings, - vm_exclusions, - dict_args, - pipeline_name, - risk_exclusions, - remote_config, - add_epss_gateway, - devops_platform_gateway, - print_table_gateway, - ) - - -def should_skip_analysis(remote_config, pipeline_name, exclusions): - ignore_pattern = remote_config["IGNORE_ANALYSIS_PATTERN"] - return re.match(ignore_pattern, pipeline_name, re.IGNORECASE) or ( - pipeline_name in exclusions and exclusions[pipeline_name].get("SKIP_TOOL", 0) - ) - - -def process_findings( - findings, - vm_exclusions, - dict_args, - pipeline_name, - risk_exclusions, - remote_config, - add_epss_gateway, - devops_platform_gateway, - print_table_gateway, -): if not findings: print("No findings found in Vulnerability Management Platform") logger.info("No findings found in Vulnerability Management Platform") @@ -78,42 +45,30 @@ def process_findings( handle_filters = HandleFilters() - return process_active_findings( - handle_filters.filter(findings), - findings, - vm_exclusions, - devops_platform_gateway, - dict_args, - remote_config, - risk_exclusions, - pipeline_name, - add_epss_gateway, - print_table_gateway, + active_findings = handle_filters.filter(findings) + + unique_findings = handle_filters.filter_duplicated(active_findings) + + filtered_findings = handle_filters.filter_tags_days( + devops_platform_gateway, remote_config, unique_findings ) + data_added = AddData(add_epss_gateway, filtered_findings).process() -def process_active_findings( - active_findings, - total_findings, - vm_exclusions, - devops_platform_gateway, - dict_args, - remote_config, - risk_exclusions, - pipeline_name, - add_epss_gateway, - print_table_gateway, -): - data_added = AddData(add_epss_gateway, active_findings).process() get_exclusions = GetExclusions( devops_platform_gateway, dict_args, data_added, remote_config, risk_exclusions, - pipeline_name, + services, ) exclusions = get_exclusions.process() + + threshold = CheckThreshold( + pipeline_name, remote_config["THRESHOLD"], risk_exclusions + ).process() + break_build = BreakBuild( devops_platform_gateway, print_table_gateway, @@ -121,7 +76,8 @@ def process_active_findings( exclusions, vm_exclusions, data_added, - total_findings, + findings, + threshold, ) return break_build.process() diff --git a/tools/devsecops_engine_tools/engine_risk/test/applications/test_runner_engine_risk.py b/tools/devsecops_engine_tools/engine_risk/test/applications/test_runner_engine_risk.py index 207e67cc6..a94563756 100644 --- a/tools/devsecops_engine_tools/engine_risk/test/applications/test_runner_engine_risk.py +++ b/tools/devsecops_engine_tools/engine_risk/test/applications/test_runner_engine_risk.py @@ -12,12 +12,14 @@ def test_runner_engine_risk(mock_init_engine_risk): print_table_gateway = "print_table_gateway" dict_args = {"key": "value"} findings = [] + services = [] vm_exclusions = [] runner_engine_risk( dict_args, findings, vm_exclusions, + services, devops_platform_gateway, print_table_gateway, ) diff --git a/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_break_build.py b/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_break_build.py index 904932def..1678e76c7 100644 --- a/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_break_build.py +++ b/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_break_build.py @@ -1,5 +1,4 @@ -import unittest -from unittest.mock import MagicMock, patch, Mock +from unittest.mock import MagicMock, patch from devsecops_engine_tools.engine_risk.src.domain.usecases.break_build import ( BreakBuild, ) @@ -56,9 +55,10 @@ def test_process( [], [], [], + {} ) break_build.break_build = True - result = break_build.process() + break_build.process() remediation_rate_control.assert_called_once() apply_exclusions.assert_called_once() @@ -80,6 +80,7 @@ def test_breaker_break(): [], [], [], + {} ) break_build.break_build = True break_build._breaker() @@ -97,6 +98,7 @@ def test_breaker_not_break(): [], [], [], + {} ) break_build.break_build = False break_build._breaker() @@ -111,23 +113,26 @@ def test_remediation_rate_control_greater(): Report(mitigated=False), ] remediation_rate_value = round((1 / 3) * 100, 3) - remote_config = {"THRESHOLD": {"REMEDIATION_RATE": 10}} - risk_threshold = remote_config["THRESHOLD"]["REMEDIATION_RATE"] + risk_threshold = 10 devops_platform_gateway = MagicMock() break_build = BreakBuild( devops_platform_gateway, MagicMock(), - remote_config, + {}, [], [], [], [], + {"REMEDIATION_RATE": { + "1": 0, + "other": 10 + }} ) break_build._remediation_rate_control(all_report) devops_platform_gateway.message.assert_called_with( "succeeded", - f"Remediation Rate {remediation_rate_value}% is greater than {risk_threshold}%", + f"Remediation rate {remediation_rate_value}% is greater than {risk_threshold}%", ) @@ -138,23 +143,25 @@ def test_remediation_rate_control_close(): Report(mitigated=False), ] remediation_rate_value = round((1 / 3) * 100, 3) - remote_config = {"THRESHOLD": {"REMEDIATION_RATE": 30}} - risk_threshold = remote_config["THRESHOLD"]["REMEDIATION_RATE"] + risk_threshold = 30 devops_platform_gateway = MagicMock() break_build = BreakBuild( devops_platform_gateway, MagicMock(), - remote_config, + {}, [], [], [], [], + {"REMEDIATION_RATE": { + "5": 30 + }}, ) break_build._remediation_rate_control(all_report) devops_platform_gateway.message.assert_called_with( "warning", - f"Remediation Rate {remediation_rate_value}% is close to {risk_threshold}%", + f"Remediation rate {remediation_rate_value}% is close to {risk_threshold}%", ) @@ -165,60 +172,27 @@ def test_remediation_rate_control_less(): Report(mitigated=False), ] remediation_rate_value = round((1 / 3) * 100, 3) - remote_config = {"THRESHOLD": {"REMEDIATION_RATE": 50}} - risk_threshold = remote_config["THRESHOLD"]["REMEDIATION_RATE"] + risk_threshold = 50 devops_platform_gateway = MagicMock() break_build = BreakBuild( devops_platform_gateway, MagicMock(), - remote_config, + {}, [], [], [], [], + {"REMEDIATION_RATE": { + "1": 0, + "other": 50 + }}, ) break_build._remediation_rate_control(all_report) devops_platform_gateway.message.assert_called_with( "error", - f"Remediation Rate {remediation_rate_value}% is less than {risk_threshold}%", - ) - - -def test_get_applied_exclusion_id(): - report = Report(id="id") - exclusion = Exclusions(id="id") - break_build = BreakBuild( - MagicMock(), - MagicMock(), - {}, - [], - [], - [], - [], - ) - break_build.exclusions = [exclusion] - result = break_build._get_applied_exclusion(report) - - assert result == exclusion - - -def test_get_applied_exclusion_vuln_id_from_tool(): - report = Report(vuln_id_from_tool="id") - exclusion = Exclusions(id="id") - break_build = BreakBuild( - MagicMock(), - MagicMock(), - {}, - [], - [], - [], - [], + f"Remediation rate {remediation_rate_value}% is less than {risk_threshold}%", ) - break_build.exclusions = [exclusion] - result = break_build._get_applied_exclusion(report) - - assert result == exclusion def test_map_applied_exclusion(): @@ -230,6 +204,10 @@ def test_map_applied_exclusion(): create_date="create_date", expired_date="expired_date", reason="reason", + vm_id="vm_id", + vm_id_url="vm_id_url", + service="service", + tags=["tags"], ) ] expected = [ @@ -240,6 +218,10 @@ def test_map_applied_exclusion(): "create_date": "create_date", "expired_date": "expired_date", "reason": "reason", + "vm_id": "vm_id", + "vm_id_url": "vm_id_url", + "service": "service", + "tags": ["tags"], } ] @@ -251,18 +233,16 @@ def test_map_applied_exclusion(): [], [], [], + {}, ) result = break_build._map_applied_exclusion(exclusions) assert result == expected -@patch( - "devsecops_engine_tools.engine_risk.src.domain.usecases.break_build.BreakBuild._get_applied_exclusion" -) -def test_apply_exclusions_vuln_id_from_tool(get_applied_exclusion): - report_list = [Report(vuln_id_from_tool="id")] - exclusions = [Exclusions(id="id")] +def test_apply_exclusions_vuln_id_from_tool(): + report_list = [Report(vuln_id_from_tool="id", where="all")] + exclusions = [Exclusions(id="id", where="all")] break_build = BreakBuild( MagicMock(), MagicMock(), @@ -271,42 +251,18 @@ def test_apply_exclusions_vuln_id_from_tool(get_applied_exclusion): [], [], [], - ) - break_build.exclusions = exclusions - - break_build._apply_exclusions(report_list) - - get_applied_exclusion.assert_called_with(report_list[0]) - - -@patch( - "devsecops_engine_tools.engine_risk.src.domain.usecases.break_build.BreakBuild._get_applied_exclusion" -) -def test_apply_exclusions_id(get_applied_exclusion): - report_list = [Report(id="id")] - exclusions = [Exclusions(id="id")] - break_build = BreakBuild( - MagicMock(), - MagicMock(), {}, - [], - [], - [], - [], ) break_build.exclusions = exclusions - break_build._apply_exclusions(report_list) + result = break_build._apply_exclusions(report_list) - get_applied_exclusion.assert_called_with(report_list[0]) + assert result == ([], exclusions) -@patch( - "devsecops_engine_tools.engine_risk.src.domain.usecases.break_build.BreakBuild._get_applied_exclusion" -) -def test_apply_exclusions_vuln_id_from_tool(get_applied_exclusion): - report_list = [Report(id="id1")] - exclusions = [Exclusions(id="id")] +def test_apply_exclusions_id(): + report_list = [Report(id="id", where="all")] + exclusions = [Exclusions(id="id", where="all")] break_build = BreakBuild( MagicMock(), MagicMock(), @@ -315,80 +271,93 @@ def test_apply_exclusions_vuln_id_from_tool(get_applied_exclusion): [], [], [], + {}, ) break_build.exclusions = exclusions - break_build._apply_exclusions(report_list) + result = break_build._apply_exclusions(report_list) - get_applied_exclusion.assert_not_called() + assert result == ([], exclusions) def test_tag_blacklist_control_error(): - report_list = [Report(vuln_id_from_tool="id1", tags=["blacklisted"], age=10)] + report_list = [ + Report( + vuln_id_from_tool="id1", + tags=["blacklisted"], + age=10, + vm_id="vm_id", + vm_id_url="vm_id_url", + ) + ] remote_config = { - "THRESHOLD": { - "TAG_BLACKLIST": ["blacklisted"], - "TAG_MAX_AGE": 5, - } + "TAG_BLACKLIST": ["blacklisted"] } - tag_age_threshold = remote_config["THRESHOLD"]["TAG_MAX_AGE"] - devops_platform_gateway = MagicMock() + tag_age_threshold = 5 + mock_devops_platform_gateway = MagicMock() break_build = BreakBuild( - devops_platform_gateway, + mock_devops_platform_gateway, MagicMock(), remote_config, [], [], [], [], + {"TAG_MAX_AGE": 5} ) break_build._tag_blacklist_control(report_list) - devops_platform_gateway.message.assert_called_once_with( + mock_devops_platform_gateway.message.assert_called_once_with( "error", - f"Report {report_list[0].vuln_id_from_tool} with tag {report_list[0].tags[0]} is blacklisted and age {report_list[0].age} is above threshold {tag_age_threshold}", + f"Report {report_list[0].vm_id} with tag {report_list[0].tags[0]} is blacklisted and age {report_list[0].age} is above threshold {tag_age_threshold}", ) def test_tag_blacklist_control_warning(): - report_list = [Report(vuln_id_from_tool="id2", tags=["blacklisted"], age=3)] + report_list = [ + Report( + vuln_id_from_tool="id2", + tags=["blacklisted"], + age=3, + vm_id="vm_id", + vm_id_url="vm_id_url", + ) + ] remote_config = { - "THRESHOLD": { - "TAG_BLACKLIST": ["blacklisted"], - "TAG_MAX_AGE": 5, - } + "TAG_BLACKLIST": ["blacklisted"] } - tag_age_threshold = remote_config["THRESHOLD"]["TAG_MAX_AGE"] - devops_platform_gateway = MagicMock() + tag_age_threshold = 5 + mock_devops_platform_gateway = MagicMock() break_build = BreakBuild( - devops_platform_gateway, + mock_devops_platform_gateway, MagicMock(), remote_config, [], [], [], [], + {"TAG_MAX_AGE": 5} ) break_build._tag_blacklist_control(report_list) - devops_platform_gateway.message.assert_called_once_with( + mock_devops_platform_gateway.message.assert_called_once_with( "warning", - f"Report {report_list[0].vuln_id_from_tool} with tag {report_list[0].tags[0]} is blacklisted but age {report_list[0].age} is below threshold {tag_age_threshold}", + f"Report {report_list[0].vm_id} with tag {report_list[0].tags[0]} is blacklisted but age {report_list[0].age} is below threshold {tag_age_threshold}", ) def test_risk_score_control_break(): report_list = [Report(severity="high", epss_score=0, age=0, tags=["tag"])] remote_config = { - "THRESHOLD": {"RISK_SCORE": 4}, "WEIGHTS": { "severity": {"high": 5}, "epss_score": 1, "age": 1, + "max_age": 1, "tags": {"tag": 1}, }, } - risk_score_threshold = remote_config["THRESHOLD"]["RISK_SCORE"] + risk_score_threshold = 4 devops_platform_gateway = MagicMock() break_build = BreakBuild( devops_platform_gateway, @@ -398,6 +367,7 @@ def test_risk_score_control_break(): [], [], [], + {"RISK_SCORE": 4} ) break_build._risk_score_control(report_list) @@ -410,15 +380,15 @@ def test_risk_score_control_break(): def test_risk_score_control_not_break(): report_list = [Report(severity="low", epss_score=0, age=0, tags=["tag"])] remote_config = { - "THRESHOLD": {"RISK_SCORE": 4}, "WEIGHTS": { "severity": {"high": 1}, "epss_score": 1, "age": 1, + "max_age": 1, "tags": {"tag": 1}, }, } - risk_score_threshold = remote_config["THRESHOLD"]["RISK_SCORE"] + risk_score_threshold = 4 devops_platform_gateway = MagicMock() break_build = BreakBuild( devops_platform_gateway, @@ -428,6 +398,7 @@ def test_risk_score_control_not_break(): [], [], [], + {"RISK_SCORE": 4} ) break_build._risk_score_control(report_list) @@ -457,6 +428,7 @@ def test_print_exclusions(): [], [], [], + {} ) break_build._print_exclusions(applied_exclusions) diff --git a/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_check_threshold.py b/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_check_threshold.py new file mode 100644 index 000000000..d4d209846 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_check_threshold.py @@ -0,0 +1,82 @@ +from unittest.mock import MagicMock, patch +from devsecops_engine_tools.engine_risk.src.domain.usecases.check_threshold import ( + CheckThreshold, +) + +def test_process_pipeline_name(): + pipeline_name = "pipeline_name_test" + threshold = { + "REMEDIATION_RATE": 1, + "RISK_SCORE": 1, + "TAG_MAX_AGE": 1 + } + risk_exclusions = { + "pipeline_name_test": { + "THRESHOLD": { + "REMEDIATION_RATE": 2, + "RISK_SCORE": 2, + "TAG_MAX_AGE": 2 + } + } + } + expected_threshold = { + "REMEDIATION_RATE": 2, + "RISK_SCORE": 2, + "TAG_MAX_AGE": 2 + } + + check_threshold = CheckThreshold(pipeline_name, threshold, risk_exclusions) + + assert check_threshold.process() == expected_threshold + +def test_process_pattern(): + pipeline_name = "pipeline_name_test" + threshold = { + "REMEDIATION_RATE": 1, + "RISK_SCORE": 1, + "TAG_MAX_AGE": 1 + } + risk_exclusions = { + "BY_PATTERN_SEARCH": { + ".*(pipeline_name).*": { + "THRESHOLD": { + "REMEDIATION_RATE": 2, + "RISK_SCORE": 2, + "TAG_MAX_AGE": 2 + } + } + } + } + expected_threshold = { + "REMEDIATION_RATE": 2, + "RISK_SCORE": 2, + "TAG_MAX_AGE": 2 + } + + check_threshold = CheckThreshold(pipeline_name, threshold, risk_exclusions) + + assert check_threshold.process() == expected_threshold + +def test_process_default(): + pipeline_name = "pipeline_name_test" + threshold = { + "REMEDIATION_RATE": 1, + "RISK_SCORE": 1, + "TAG_MAX_AGE": 1 + } + risk_exclusions = { + "THRESHOLD": { + "REMEDIATION_RATE": 2, + "RISK_SCORE": 2, + "TAG_MAX_AGE": 2 + } + } + expected_threshold = { + "REMEDIATION_RATE": 1, + "RISK_SCORE": 1, + "TAG_MAX_AGE": 1 + } + + check_threshold = CheckThreshold(pipeline_name, threshold, risk_exclusions) + + assert check_threshold.process() == expected_threshold \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_get_exclusions.py b/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_get_exclusions.py index 84a398e41..d3cd357cb 100644 --- a/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_get_exclusions.py +++ b/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_get_exclusions.py @@ -20,7 +20,7 @@ def test_process( get_exclusions = GetExclusions( MagicMock(), - {"remote_config_repo": "repo"}, + {"remote_config_repo": "repo", "remote_config_branch": "repo"}, MagicMock(), {"EXCLUSIONS_PATHS": {"tag1": "path1"}}, MagicMock(), @@ -95,7 +95,7 @@ def test_get_exclusions(mock_exclusions): MagicMock(), MagicMock(), MagicMock(), - "pipeline_name", + ["service1", "service2"], ) exclusions = get_exclusions._get_exclusions(config, "RISK") diff --git a/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_handle_filters.py b/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_handle_filters.py index 5a54ef067..0b9997f65 100644 --- a/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_handle_filters.py +++ b/tools/devsecops_engine_tools/engine_risk/test/domain/usecases/test_handle_filters.py @@ -58,6 +58,84 @@ def test_filter(self, mock_get_priority_vulnerability, mock_get_active_findings) assert len(result) == 2 + def test_filter_duplicated(self): + findings = [ + Report( + id=["CVE-2021-1234"], + date="21022024", + status="stat2", + where="path2", + tags=["tag1"], + severity="low", + active=True, + ), + Report( + id=["CVE-2021-1234"], + date="21022024", + status="stat2", + where="path2", + tags=["tag2"], + severity="low", + active=None, + ), + Report( + id=["vuln_id"], + date="21022024", + status="stat3", + where="path3", + tags=["tag3"], + severity="info", + active=True, + ), + ] + + result = self.handle_filters.filter_duplicated(findings) + + assert len(result) == 2 + + def test_filter_tags_days(self): + remote_config = {"TAG_EXCLUSION_DAYS": {"tag1": 5, "tag2": 10}} + findings = [ + Report( + id=["CVE-2021-1234"], + date="21022024", + status="stat2", + where="path2", + tags=["tag1"], + severity="low", + active=True, + age=4, + ), + Report( + id=["CVE-2021-1234"], + date="21022024", + status="stat2", + where="path2", + tags=["tag2"], + severity="low", + active=None, + age=11, + ), + Report( + id=["vuln_id"], + date="21022024", + status="stat3", + where="path3", + tags=["tag3"], + severity="info", + active=True, + age=5, + ), + ] + devops_platform_gateway = MagicMock() + + result = self.handle_filters.filter_tags_days( + devops_platform_gateway, remote_config, findings + ) + + devops_platform_gateway.message.assert_called_once() + assert len(result) == 2 + def test__get_active_findings(self): result = self.handle_filters._get_active_findings(self.findings) diff --git a/tools/devsecops_engine_tools/engine_risk/test/infrastructure/entry_points/test_entry_point_risk.py b/tools/devsecops_engine_tools/engine_risk/test/infrastructure/entry_points/test_entry_point_risk.py index 93e8242cf..cd9dee776 100644 --- a/tools/devsecops_engine_tools/engine_risk/test/infrastructure/entry_points/test_entry_point_risk.py +++ b/tools/devsecops_engine_tools/engine_risk/test/infrastructure/entry_points/test_entry_point_risk.py @@ -1,124 +1,71 @@ from unittest.mock import MagicMock, patch from devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk import ( init_engine_risk, - should_skip_analysis, - process_findings, ) @patch( - "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.should_skip_analysis" + "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.HandleFilters" ) -@patch("builtins.print") -def test_init_engine_risk_skip(mock_print, mock_skip): - dict_args = {"remote_config_repo": "remote_config"} - findings = ["finding1", "finding2"] - vm_exclusions = ["exclusion1", "exclusion2"] - mock_skip.return_value = True - - init_engine_risk( - MagicMock(), MagicMock(), MagicMock(), dict_args, findings, vm_exclusions - ) - - mock_print.assert_called_once_with("Tool skipped by DevSecOps Policy.") - - @patch( - "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.should_skip_analysis" + "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.BreakBuild" ) @patch( - "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.process_findings" + "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.AddData" ) -def test_init_engine_risk_process(mock_process, mock_skip): - dict_args = {"remote_config_repo": "remote_config"} +@patch( + "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.GetExclusions" +) +def test_init_engine_risk( + mock_get_exclusions, mock_add_data, mock_break_build, mock_handle_filters +): + dict_args = {"remote_config_repo": "remote_config", "remote_config_branch": ""} findings = ["finding1", "finding2"] + services = ["service1", "service2"] vm_exclusions = ["exclusion1", "exclusion2"] - mock_skip.return_value = False + mock_devops_platform_gateway = MagicMock() + mock_print_table_gateway = MagicMock() init_engine_risk( - MagicMock(), MagicMock(), MagicMock(), dict_args, findings, vm_exclusions - ) - - mock_process.assert_called_once() - - -def test_should_skip_analysis(): - remote_config = {"IGNORE_ANALYSIS_PATTERN": "pattern"} - pipeline_name = "pipeline" - exclusions = {"pipeline": {"SKIP_TOOL": 1}} - - assert should_skip_analysis(remote_config, pipeline_name, exclusions) == True - - -@patch("builtins.print") -def test_process_findings_no_findings(mock_print): - findings = [] - - process_findings( - findings, - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - ) - - mock_print.assert_called_once_with( - "No findings found in Vulnerability Management Platform" - ) - - -@patch( - "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.HandleFilters" -) -@patch( - "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.process_active_findings" -) -def test_process_findings(mock_process_active, mock_filters): - findings = ["finding1", "finding2"] - mock_filters.return_value.filter.return_value = [] - - process_findings( + mock_devops_platform_gateway, + mock_print_table_gateway, + dict_args, findings, - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), + services, + vm_exclusions, ) - mock_process_active.assert_called_once() + assert mock_devops_platform_gateway.get_remote_config.call_count == 2 + mock_handle_filters.return_value.filter.assert_called_once_with(findings) + mock_handle_filters.return_value.filter_duplicated.assert_called_once() + mock_handle_filters.return_value.filter_tags_days.assert_called_once() + mock_add_data.assert_called_once() + mock_get_exclusions.assert_called_once() + mock_break_build.assert_called_once() @patch( - "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.GetExclusions" -) -@patch( - "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.AddData" + "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.logger" ) -@patch( - "devsecops_engine_tools.engine_risk.src.infrastructure.entry_points.entry_point_risk.BreakBuild" -) -def test_process_active_findings(mock_break, mock_add, mock_exclusions): +def test_init_engine_risk_no_findings(mock_logger): + dict_args = {"remote_config_repo": "remote_config", "remote_config_branch": ""} + findings = [] + services = ["service1", "service2"] + vm_exclusions = ["exclusion1", "exclusion2"] + mock_devops_platform_gateway = MagicMock() + mock_print_table_gateway = MagicMock() - process_findings( - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), + init_engine_risk( MagicMock(), + mock_devops_platform_gateway, + mock_print_table_gateway, + dict_args, + findings, + services, + vm_exclusions, ) - mock_add.return_value.process.assert_called_once() - mock_exclusions.return_value.process.assert_called_once() - mock_break.return_value.process.assert_called_once() + mock_logger.info.assert_called_once_with( + "No findings found in Vulnerability Management Platform" + ) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_code/src/domain/usecases/code_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_code/src/domain/usecases/code_scan.py index 5af92da75..83f563ba0 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_code/src/domain/usecases/code_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_code/src/domain/usecases/code_scan.py @@ -6,15 +6,14 @@ DevopsPlatformGateway, ) from devsecops_engine_tools.engine_utilities.git_cli.model.gateway.git_gateway import ( - GitGateway + GitGateway, ) from devsecops_engine_tools.engine_sast.engine_code.src.domain.model.config_tool import ( ConfigTool, ) from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions -from devsecops_engine_tools.engine_core.src.domain.model.input_core import ( - InputCore -) +from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore +from devsecops_engine_tools.engine_utilities.utils.utils import Utils from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings @@ -23,7 +22,10 @@ class CodeScan: def __init__( - self, tool_gateway: ToolGateway, devops_platform_gateway: DevopsPlatformGateway, git_gateway: GitGateway + self, + tool_gateway: ToolGateway, + devops_platform_gateway: DevopsPlatformGateway, + git_gateway: GitGateway, ): self.tool_gateway = tool_gateway self.devops_platform_gateway = devops_platform_gateway @@ -31,14 +33,11 @@ def __init__( def set_config_tool(self, dict_args): init_config_tool = self.devops_platform_gateway.get_remote_config( - dict_args["remote_config_repo"], - "engine_sast/engine_code/ConfigTool.json" - ) - scope_pipeline = self.devops_platform_gateway.get_variable( - "pipeline_name" + dict_args["remote_config_repo"], "engine_sast/engine_code/ConfigTool.json", dict_args["remote_config_branch"] ) + scope_pipeline = self.devops_platform_gateway.get_variable("pipeline_name") return ConfigTool(json_data=init_config_tool, scope=scope_pipeline) - + def get_pull_request_files(self, target_branches): files_pullrequest = self.git_gateway.get_files_pull_request( self.devops_platform_gateway.get_variable("path_directory"), @@ -49,19 +48,17 @@ def get_pull_request_files(self, target_branches): self.devops_platform_gateway.get_variable("organization"), self.devops_platform_gateway.get_variable("project_name"), self.devops_platform_gateway.get_variable("repository"), - self.devops_platform_gateway.get_variable("repository_provider") - ) + self.devops_platform_gateway.get_variable("repository_provider"), + ) return files_pullrequest - def get_exclusions(self, dict_args, tool): - exclusions_data = self.devops_platform_gateway.get_remote_config( - dict_args["remote_config_repo"], - "engine_sast/engine_code/Exclusions.json" - ) + def get_exclusions(self, tool, exclusions_data): list_exclusions = [] skip_tool = False for pipeline, exclusions in exclusions_data.items(): - if (pipeline == "All") or (pipeline == self.devops_platform_gateway.get_variable("pipeline_name")): + if (pipeline == "All") or ( + pipeline == self.devops_platform_gateway.get_variable("pipeline_name") + ): if exclusions.get("SKIP_TOOL", False): skip_tool = True elif exclusions.get(tool, False): @@ -78,10 +75,12 @@ def get_exclusions(self, dict_args, tool): list_exclusions.append(exclusion) return list_exclusions, skip_tool - def apply_exclude_path(self, exclude_folder, ignore_search_pattern, pull_request_file): + def apply_exclude_path( + self, exclude_folder, ignore_search_pattern, pull_request_file + ): patterns = ignore_search_pattern patterns.extend([rf"/{re.escape(folder)}//*" for folder in exclude_folder]) - + for pattern in patterns: if re.search(pattern, pull_request_file): return True @@ -89,35 +88,54 @@ def apply_exclude_path(self, exclude_folder, ignore_search_pattern, pull_request def process(self, dict_args, tool): config_tool = self.set_config_tool(dict_args) - list_exclusions, skip_tool = self.get_exclusions(dict_args, tool) + exclusions_data = self.devops_platform_gateway.get_remote_config( + dict_args["remote_config_repo"], "engine_sast/engine_code/Exclusions.json" + ) + list_exclusions, skip_tool = self.get_exclusions(tool, exclusions_data) findings_list, path_file_results = [], "" if not skip_tool: pull_request_files = [] if not dict_args["folder_path"]: - pull_request_files = self.get_pull_request_files(config_tool.target_branches) - pull_request_files = [pf for pf in pull_request_files - if not self.apply_exclude_path(config_tool.exclude_folder, config_tool.ignore_search_pattern, pf)] + pull_request_files = self.get_pull_request_files( + config_tool.target_branches + ) + pull_request_files = [ + pf + for pf in pull_request_files + if not self.apply_exclude_path( + config_tool.exclude_folder, + config_tool.ignore_search_pattern, + pf, + ) + ] findings_list, path_file_results = self.tool_gateway.run_tool( - dict_args["folder_path"], + dict_args["folder_path"], pull_request_files, self.devops_platform_gateway.get_variable("path_directory"), self.devops_platform_gateway.get_variable("repository"), - config_tool + config_tool, ) else: - print(f"Tool skipped by DevSecOps policy") - logger.info(f"Tool skipped by DevSecOps policy") + print("Tool skipped by DevSecOps policy") + dict_args["send_metrics"] = "false" input_core = InputCore( totalized_exclusions=list_exclusions, - threshold_defined=config_tool.threshold, + threshold_defined=Utils.update_threshold( + self, + config_tool.threshold, + exclusions_data, + config_tool.scope_pipeline, + ), path_file_results=path_file_results, custom_message_break_build=config_tool.message_info_engine_code, scope_pipeline=config_tool.scope_pipeline, - stage_pipeline=self.devops_platform_gateway.get_variable("stage").capitalize(), + stage_pipeline=self.devops_platform_gateway.get_variable( + "stage" + ).capitalize(), ) return findings_list, input_core diff --git a/tools/devsecops_engine_tools/engine_sast/engine_code/test/domain/usecases/test_code_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_code/test/domain/usecases/test_code_scan.py index 42382d571..b46d4a190 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_code/test/domain/usecases/test_code_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_code/test/domain/usecases/test_code_scan.py @@ -39,11 +39,11 @@ def test_set_config_tool(self, mock_config_tool): self.mock_devops_platform_gateway.get_variable.return_value = "pipeline_test_name" # Act - self.code_scan.set_config_tool({"remote_config_repo": "test_repo"}) + self.code_scan.set_config_tool({"remote_config_repo": "test_repo", "remote_config_branch": ""}) # Assert self.mock_devops_platform_gateway.get_remote_config.assert_called_once_with( - "test_repo", "engine_sast/engine_code/ConfigTool.json" + "test_repo", "engine_sast/engine_code/ConfigTool.json", "" ) self.mock_devops_platform_gateway.get_variable.assert_called_once_with("pipeline_name") mock_config_tool.assert_called_once_with( @@ -71,12 +71,9 @@ def test_get_pull_request_files(self): ) self.assertEqual(files, ["file1", "file2"]) - @patch( - "devsecops_engine_tools.engine_sast.engine_code.src.domain.usecases.code_scan.Exclusions" - ) - def test_get_exclusions_all_pipelines(self, mock_exclusions): + def test_get_exclusions_all_pipelines(self): # Arrange - self.mock_devops_platform_gateway.get_remote_config.return_value = { + exclusions_data = { "All":{ "TOOL_NAME": [ { @@ -90,27 +87,18 @@ def test_get_exclusions_all_pipelines(self, mock_exclusions): } } self.mock_devops_platform_gateway.get_variable.return_value = "pipeline_test_name" - mock_exclusions.return_value = Mock() # Act - exclusions, skip_tool = self.code_scan.get_exclusions({"remote_config_repo": "test_repo"}, "TOOL_NAME") + exclusions, skip_tool = self.code_scan.get_exclusions("TOOL_NAME", exclusions_data) # Aserciones - self.mock_devops_platform_gateway.get_remote_config.assert_called_once_with( - "test_repo", "engine_sast/engine_code/Exclusions.json" - ) self.assertFalse(skip_tool) - mock_exclusions.assert_called_with( - id="vul id", where="", create_date="18102023", expired_date="18042024", severity="low", hu="4338704", reason="Risk acceptance" - ) self.assertEqual(len(exclusions), 1) - @patch( - "devsecops_engine_tools.engine_sast.engine_code.src.domain.usecases.code_scan.Exclusions" - ) - def test_get_exclusions_specific_pipeline(self, mock_exclusions): + + def test_get_exclusions_specific_pipeline(self): # Arrange - self.mock_devops_platform_gateway.get_remote_config.return_value = { + exclusions_data = { "pipeline_test_name":{ "TOOL_NAME": [ { @@ -124,27 +112,18 @@ def test_get_exclusions_specific_pipeline(self, mock_exclusions): } } self.mock_devops_platform_gateway.get_variable.return_value = "pipeline_test_name" - mock_exclusions.return_value = Mock() # Act - exclusions, skip_tool = self.code_scan.get_exclusions({"remote_config_repo": "test_repo"}, "TOOL_NAME") + exclusions, skip_tool = self.code_scan.get_exclusions("TOOL_NAME", exclusions_data) # Assert - self.mock_devops_platform_gateway.get_remote_config.assert_called_once_with( - "test_repo", "engine_sast/engine_code/Exclusions.json" - ) self.assertFalse(skip_tool) - mock_exclusions.assert_called_with( - id="vul id", where="", create_date="18102023", expired_date="18042024", severity="low", hu="4338704", reason="Risk acceptance" - ) self.assertEqual(len(exclusions), 1) - @patch( - "devsecops_engine_tools.engine_sast.engine_code.src.domain.usecases.code_scan.Exclusions" - ) - def test_get_exclusions_skip_tool(self, mock_exclusions): + + def test_get_exclusions_skip_tool(self): # Arrange - self.mock_devops_platform_gateway.get_remote_config.return_value = { + exclusions_data = { "pipeline_test_name":{ "SKIP_TOOL": { "create_date": "24012024", @@ -154,15 +133,11 @@ def test_get_exclusions_skip_tool(self, mock_exclusions): } } self.mock_devops_platform_gateway.get_variable.return_value = "pipeline_test_name" - mock_exclusions.return_value = Mock() # Act - exclusions, skip_tool = self.code_scan.get_exclusions({"remote_config_repo": "test_repo"}, "TOOL_NAME") + exclusions, skip_tool = self.code_scan.get_exclusions("TOOL_NAME", exclusions_data) # Assert - self.mock_devops_platform_gateway.get_remote_config.assert_called_once_with( - "test_repo", "engine_sast/engine_code/Exclusions.json" - ) self.assertTrue(skip_tool) self.assertEqual(exclusions, []) @@ -202,7 +177,7 @@ def test_process(self, mock_input_core): self.mock_devops_platform_gateway.get_variable.side_effect = ["test_work_folder", "test_repo", "test_stage"] # Act - findings_list, _ = self.code_scan.process({"folder_path": None, "remote_config_repo": "some_repo"}, "TOOL_NAME") + findings_list, _ = self.code_scan.process({"folder_path": None, "remote_config_repo": "some_repo", "remote_config_branch": ""}, "TOOL_NAME") # Assert self.code_scan.set_config_tool.assert_called_once() @@ -228,7 +203,7 @@ def test_process_skip_tool(self, mock_input_core): self.mock_devops_platform_gateway.get_variable.return_value = "test_stage" # Act - findings_list, _ = self.code_scan.process({"folder_path": None, "remote_config_repo": "some_repo"}, "TOOL_NAME") + findings_list, _ = self.code_scan.process({"folder_path": None, "remote_config_repo": "some_repo", "remote_config_branch": ""}, "TOOL_NAME") # Assert self.code_scan.set_config_tool.assert_called_once() diff --git a/tools/devsecops_engine_tools/engine_sast/engine_iac/src/domain/usecases/iac_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_iac/src/domain/usecases/iac_scan.py index 50acb9d2b..be8f8ced5 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_iac/src/domain/usecases/iac_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_iac/src/domain/usecases/iac_scan.py @@ -13,6 +13,8 @@ from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings +from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold +from devsecops_engine_tools.engine_utilities.utils.utils import Utils logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() @@ -26,11 +28,11 @@ def __init__( def process(self, dict_args, secret_tool, tool, env): config_tool_iac = self.devops_platform_gateway.get_remote_config( - dict_args["remote_config_repo"], "engine_sast/engine_iac/ConfigTool.json" + dict_args["remote_config_repo"], "engine_sast/engine_iac/ConfigTool.json", dict_args["remote_config_branch"] ) exclusions = self.devops_platform_gateway.get_remote_config( - dict_args["remote_config_repo"], "engine_sast/engine_iac/Exclusions.json" + dict_args["remote_config_repo"], "engine_sast/engine_iac/Exclusions.json", dict_args["remote_config_branch"] ) config_tool_core, folders_to_scan, skip_tool = self.complete_config_tool( @@ -45,11 +47,11 @@ def process(self, dict_args, secret_tool, tool, env): environment="pdn" if env not in ["dev", "qa", "pdn"] else env, platform_to_scan=dict_args["platform"], secret_tool=secret_tool, - secret_external_checks=dict_args["token_external_checks"] + secret_external_checks=dict_args["token_external_checks"], ) else: - print(f"Tool skipped by DevSecOps policy") - logger.info(f"Tool skipped by DevSecOps policy") + print("Tool skipped by DevSecOps policy") + dict_args["send_metrics"] = "false" totalized_exclusions = [] ( @@ -69,7 +71,12 @@ def process(self, dict_args, secret_tool, tool, env): input_core = InputCore( totalized_exclusions=totalized_exclusions, - threshold_defined=config_tool_core.threshold, + threshold_defined=Utils.update_threshold( + self, + config_tool_core.threshold, + exclusions, + config_tool_core.scope_pipeline, + ), path_file_results=path_file_results, custom_message_break_build=config_tool_core.message_info_engine_iac, scope_pipeline=config_tool_core.scope_pipeline, @@ -88,7 +95,13 @@ def complete_config_tool(self, data_file_tool, exclusions, tool, dict_args): "pipeline_name" ) - skip_tool = bool(re.match(config_tool.ignore_search_pattern, config_tool.scope_pipeline, re.IGNORECASE)) + skip_tool = bool( + re.match( + config_tool.ignore_search_pattern, + config_tool.scope_pipeline, + re.IGNORECASE, + ) + ) if config_tool.exclusions.get("All") is not None: config_tool.exclusions_all = config_tool.exclusions.get("All").get(tool) @@ -96,11 +109,13 @@ def complete_config_tool(self, data_file_tool, exclusions, tool, dict_args): config_tool.exclusions_scope = config_tool.exclusions.get( config_tool.scope_pipeline ).get(tool) - skip_tool = bool(config_tool.exclusions.get(config_tool.scope_pipeline).get("SKIP_TOOL")) + skip_tool = bool( + config_tool.exclusions.get(config_tool.scope_pipeline).get("SKIP_TOOL") + ) if dict_args["folder_path"]: if ( - config_tool.update_service_file_name_cft == "True" + config_tool.update_service_file_name_cft and "cloudformation" in dict_args["platform"] ): files = os.listdir(os.path.join(os.getcwd(), dict_args["folder_path"])) @@ -124,11 +139,7 @@ def complete_config_tool(self, data_file_tool, exclusions, tool, dict_args): def search_folders(self, search_pattern): current_directory = os.getcwd() - patron = ( - "(?i).*?(" - + "|".join(search_pattern) - + ").*$" - ) + patron = "(?i).*?(" + "|".join(search_pattern) + ").*$" folders = [ folder for folder in os.listdir(current_directory) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_deserealizator.py b/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_deserealizator.py index 30e50f04c..2539f5736 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_deserealizator.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_deserealizator.py @@ -10,23 +10,33 @@ class CheckovDeserealizator: @classmethod def get_list_finding( - cls, results_scan_list: list, rules + cls, results_scan_list: list, rules, default_severity, default_category ) -> "list[Finding]": - list_open_findings = [] + list_open_findings = [] for result in results_scan_list: if "failed_checks" in str(result): for scan in result["results"]["failed_checks"]: + check_id = scan.get("check_id") + if not rules.get(check_id): + description = scan.get("check_name") + severity = default_severity.lower() + category = default_category.lower() + else: + description = rules[check_id].get("checkID", scan.get("check_name")) + severity = rules[check_id].get("severity").lower() + category = rules[check_id].get("category").lower() + finding_open = Finding( - id=scan.get("check_id"), + id=check_id, cvss=None, - where = scan.get("repo_file_path") + ": " + str(scan.get("resource")), - description=rules[scan.get("check_id")].get("checkID", scan.get("check_name")), - severity=rules[scan.get("check_id")].get("severity").lower(), + where=scan.get("repo_file_path") + ": " + str(scan.get("resource")), + description=description, + severity=severity, identification_date=datetime.now().strftime("%d%m%Y"), published_date_cve=None, module="engine_iac", - category=Category(rules[scan.get("check_id")].get("category").lower()), + category=Category(category), requirements=scan.get("guideline"), tool="Checkov" ) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_tool.py old mode 100644 new mode 100755 index ba8640934..60b72b110 --- a/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_tool.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_tool.py @@ -2,7 +2,6 @@ import subprocess import time import os -import platform import queue import threading import json @@ -19,22 +18,15 @@ from devsecops_engine_tools.engine_sast.engine_iac.src.infrastructure.helpers.file_generator_tool import ( generate_file_from_tool, ) -from devsecops_engine_tools.engine_utilities.github.infrastructure.github_api import ( - GithubApi, -) -from devsecops_engine_tools.engine_utilities.ssh.managment_private_key import ( - create_ssh_private_file, - add_ssh_private_key, - decode_base64, - config_knowns_hosts, -) from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings +from devsecops_engine_tools.engine_utilities.utils.utils import Utils logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() class CheckovTool(ToolGateway): + CHECKOV_CONFIG_FILE = "checkov_config.yaml" TOOL_CHECKOV = "CHECKOV" framework_mapping = { @@ -42,12 +34,14 @@ class CheckovTool(ToolGateway): "RULES_K8S": "kubernetes", "RULES_CLOUDFORMATION": "cloudformation", "RULES_OPENAPI": "openapi", + "RULES_TERRAFORM": "terraform" } framework_external_checks = [ "RULES_K8S", "RULES_CLOUDFORMATION", "RULES_DOCKER", "RULES_OPENAPI", + "RULES_TERRAFORM" ] def create_config_file(self, checkov_config: CheckovConfig): @@ -60,43 +54,6 @@ def create_config_file(self, checkov_config: CheckovConfig): yaml.dump(checkov_config.dict_confg_file, file) file.close() - def configurate_external_checks(self, config_tool, secret): - agent_env = None - try: - if secret is None: - logger.warning("The secret is not configured for external controls") - - # Create configuration git external checks - elif config_tool[self.TOOL_CHECKOV][ - "USE_EXTERNAL_CHECKS_GIT" - ] == "True" and platform.system() in ( - "Linux", - "Darwin", - ): - config_knowns_hosts( - config_tool[self.TOOL_CHECKOV]["EXTERNAL_GIT_SSH_HOST"], - config_tool[self.TOOL_CHECKOV][ - "EXTERNAL_GIT_PUBLIC_KEY_FINGERPRINT" - ], - ) - ssh_key_content = decode_base64(secret["repository_ssh_private_key"]) - ssh_key_file_path = "/tmp/ssh_key_file" - create_ssh_private_file(ssh_key_file_path, ssh_key_content) - ssh_key_password = decode_base64(secret["repository_ssh_password"]) - agent_env = add_ssh_private_key(ssh_key_file_path, ssh_key_password) - - # Create configuration dir external checks - elif config_tool[self.TOOL_CHECKOV]["USE_EXTERNAL_CHECKS_DIR"] == "True": - github_api = GithubApi(secret["github_token"]) - github_api.download_latest_release_assets( - config_tool[self.TOOL_CHECKOV]["EXTERNAL_DIR_OWNER"], - config_tool[self.TOOL_CHECKOV]["EXTERNAL_DIR_REPOSITORY"], - "/tmp", - ) - - except Exception as ex: - logger.error(f"An error ocurred configuring external checks {ex}") - return agent_env def retryable_install_package(self, package: str, version: str) -> bool: MAX_RETRIES = 3 @@ -191,10 +148,14 @@ def scan_folders( if "all" in platform_to_scan or any( elem.upper() in rule for elem in platform_to_scan ): + framework = [self.framework_mapping[rule]] + if "terraform" in platform_to_scan or ("all" in platform_to_scan and self.framework_mapping[rule] == "terraform"): + framework.append("terraform_plan") + checkov_config = CheckovConfig( path_config_file="", config_file_name=rule, - framework=self.framework_mapping[rule], + framework=framework, checks=[ key for key, value in config_tool[self.TOOL_CHECKOV]["RULES"][ @@ -209,7 +170,6 @@ def scan_folders( f"{config_tool[self.TOOL_CHECKOV]['EXTERNAL_CHECKS_GIT']}/{self.framework_mapping[rule]}" ] if config_tool[self.TOOL_CHECKOV]["USE_EXTERNAL_CHECKS_GIT"] - == "True" and agent_env is not None and rule in self.framework_external_checks else [] @@ -218,7 +178,6 @@ def scan_folders( external_checks_dir=( f"/tmp/rules/{self.framework_mapping[rule]}" if config_tool[self.TOOL_CHECKOV]["USE_EXTERNAL_CHECKS_DIR"] - == "True" and rule in self.framework_external_checks else [] ), @@ -252,29 +211,8 @@ def run_tool( secret_tool, secret_external_checks, ): - secret = None - if secret_tool is not None: - secret = secret_tool - elif secret_external_checks is not None: - secret = { - "github_token": ( - secret_external_checks.split("github:")[1] - if "github" in secret_external_checks - else None - ), - "repository_ssh_private_key": ( - secret_external_checks.split("ssh:")[1].split(":")[0] - if "ssh" in secret_external_checks - else None - ), - "repository_ssh_password": ( - secret_external_checks.split("ssh:")[1].split(":")[1] - if "ssh" in secret_external_checks - else None - ), - } - - agent_env = self.configurate_external_checks(config_tool, secret) + util = Utils() + agent_env = util.configurate_external_checks(self.TOOL_CHECKOV,config_tool, secret_tool,secret_external_checks) checkov_install = self.retryable_install_package( "checkov", config_tool[self.TOOL_CHECKOV]["VERSION"] @@ -287,12 +225,21 @@ def run_tool( checkov_deserealizator = CheckovDeserealizator() findings_list = checkov_deserealizator.get_list_finding( - result_scans, rules_run + result_scans, + rules_run, + config_tool[self.TOOL_CHECKOV]["DEFAULT_SEVERITY"], + config_tool[self.TOOL_CHECKOV]["DEFAULT_CATEGORY"] ) return ( findings_list, - generate_file_from_tool(self.TOOL_CHECKOV, result_scans, rules_run), + generate_file_from_tool( + self.TOOL_CHECKOV, + result_scans, + rules_run, + config_tool[self.TOOL_CHECKOV]["DEFAULT_SEVERITY"], + config_tool[self.TOOL_CHECKOV]["DEFAULT_CATEGORY"] + ), ) else: - return [], None + return [], None \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/helpers/file_generator_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/helpers/file_generator_tool.py index 38c12cafb..829fc94eb 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/helpers/file_generator_tool.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/helpers/file_generator_tool.py @@ -6,7 +6,7 @@ logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() -def generate_file_from_tool(tool, result_list, rules_doc): +def generate_file_from_tool(tool, result_list, rules_doc, default_severity, default_category): if tool == "CHECKOV": try: if len(result_list) > 0: @@ -20,7 +20,7 @@ def generate_file_from_tool(tool, result_list, rules_doc): for result in result_list: failed_checks = result.get("results", {}).get("failed_checks", []) all_failed_checks.extend( - map(lambda x: update_fields(x, rules_doc), failed_checks) + map(lambda x: update_fields(x, rules_doc, default_severity, default_category), failed_checks) ) summary_passed += result.get("summary", {}).get("passed", 0) summary_failed += result.get("summary", {}).get("failed", 0) @@ -60,15 +60,14 @@ def generate_file_from_tool(tool, result_list, rules_doc): logger.error(f"Error during handling checkov json integrator {ex}") -def update_fields(check_result, rules_doc): +def update_fields(check_result, rules_doc, default_severity, default_category): rule_info = rules_doc.get(check_result.get("check_id"), {}) - check_result["severity"] = rule_info["severity"].lower() + check_result["severity"] = rule_info.get("severity", default_severity) + check_result["bc_category"] = rule_info.get("category", default_category) if "customID" in rule_info: check_result["custom_vuln_id"] = rule_info["customID"] if "guideline" in rule_info: check_result["guideline"] = rule_info["guideline"] - if "category" in rule_info: - check_result["bc_category"] = rule_info["category"] return check_result diff --git a/tools/devsecops_engine_tools/engine_sast/engine_iac/test/domain/usecases/test_iac_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_iac/test/domain/usecases/test_iac_scan.py index 5e2eb600e..fc05d5a8b 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_iac/test/domain/usecases/test_iac_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_iac/test/domain/usecases/test_iac_scan.py @@ -20,6 +20,7 @@ def side_effect(self, arg): def test_process(self): dict_args = { "remote_config_repo": "example_repo", + "remote_config_branch": "", "folder_path": ".", "environment": "test", "platform": "cloudformation", @@ -78,6 +79,7 @@ def test_process(self): def test_process_skip_search_folder(self): dict_args = { "remote_config_repo": "example_repo", + "remote_config_branch": "", "folder_path": "example_folder", "environment": "test", "platform": "eks", diff --git a/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/driven_adapters/checkov/test_checkov_deserealizator.py b/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/driven_adapters/checkov/test_checkov_deserealizator.py index e105cf6c6..9f4576a55 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/driven_adapters/checkov/test_checkov_deserealizator.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/driven_adapters/checkov/test_checkov_deserealizator.py @@ -93,7 +93,7 @@ def test_get_list_finding(): } list_findings = CheckovDeserealizator.get_list_finding( - results_scan_list, config_rules + results_scan_list, config_rules, "", "" ) list_findings_compare: list[Finding] = [] diff --git a/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/driven_adapters/checkov/test_checkov_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/driven_adapters/checkov/test_checkov_tool.py old mode 100644 new mode 100755 index 7604698fe..4d954e1dc --- a/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/driven_adapters/checkov/test_checkov_tool.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/driven_adapters/checkov/test_checkov_tool.py @@ -1,6 +1,6 @@ -import unittest -from unittest.mock import MagicMock from unittest import mock +import pytest +from unittest.mock import MagicMock, patch from queue import Queue from devsecops_engine_tools.engine_sast.engine_iac.src.infrastructure.driven_adapters.checkov.checkov_tool import ( CheckovTool, @@ -8,272 +8,115 @@ import os -class TestCheckovTool(unittest.TestCase): - def setUp(self): - self.checkov_tool = CheckovTool() +@pytest.fixture +def checkov_tool(): + return CheckovTool() - def test_create_config_file(self): - checkov_config = MagicMock() - checkov_config.path_config_file = "/path/to/config/" - checkov_config.config_file_name = "docker" - checkov_config.dict_confg_file = {"key": "value"} +def test_create_config_file(checkov_tool): + checkov_config = MagicMock() + checkov_config.path_config_file = "/path/to/config/" + checkov_config.config_file_name = "docker" + checkov_config.dict_confg_file = {"key": "value"} - with mock.patch("builtins.open", create=True) as mock_open: - self.checkov_tool.create_config_file(checkov_config) + with patch("builtins.open", create=True) as mock_open: + checkov_tool.create_config_file(checkov_config) - mock_open.assert_called_once_with( - "/path/to/config/dockercheckov_config.yaml", "w" - ) - - def test_configurate_external_checks_git(self): - # Configurar valores simulados - json_data = { - "SEARCH_PATTERN": ["AW", "NU"], - "IGNORE_SEARCH_PATTERN": [ - "test", - ], - "MESSAGE_INFO_ENGINE_IAC": "message test", - "EXCLUSIONS_PATH": "Exclusions.json", - "UPDATE_SERVICE_WITH_FILE_NAME_CFT": "false", - "THRESHOLD": { - "VULNERABILITY": { - "Critical": 10, - "High": 3, - "Medium": 20, - "Low": 30, - }, - "COMPLIANCE": {"Critical": 4}, - }, - "CHECKOV": { - "VERSION": "2.3.296", - "USE_EXTERNAL_CHECKS_GIT": "True", - "EXTERNAL_CHECKS_GIT": "rules", - "EXTERNAL_GIT_SSH_HOST": "github", - "EXTERNAL_GIT_PUBLIC_KEY_FINGERPRINT": "fingerprint", - "USE_EXTERNAL_CHECKS_DIR": "False", - "EXTERNAL_DIR_OWNER": "test", - "EXTERNAL_DIR_REPOSITORY": "repository", - "EXTERNAL_DIR_ASSET_NAME": "rules", - "RULES": "", - }, - } - mock_secret_tool = { - "repository_ssh_private_key": "cmVwb3NpdG9yeV9zc2hfcHJpdmF0ZV9rZXkK", - "repository_ssh_password": "cmVwb3NpdG9yeV9zc2hfcGFzc3dvcmQK", - } - - # Llamar al método que se está probando - result = self.checkov_tool.configurate_external_checks( - json_data, mock_secret_tool + mock_open.assert_called_once_with( + "/path/to/config/dockercheckov_config.yaml", "w" ) - # Verificar que el resultado es el esperado - self.assertIsNone(result) - - @mock.patch( - "devsecops_engine_tools.engine_utilities.github.infrastructure.github_api.GithubApi.download_latest_release_assets", - autospec=True, - ) - def test_configurate_external_checks_dir(self, mock_github_api): - # Configurar valores simulados - json_data = { - "SEARCH_PATTERN": ["AW", "NU"], - "IGNORE_SEARCH_PATTERN": [ - "test", - ], - "MESSAGE_INFO_ENGINE_IAC": "message test", - "EXCLUSIONS_PATH": "Exclusions.json", - "UPDATE_SERVICE_WITH_FILE_NAME_CFT": "false", - "THRESHOLD": { - "VULNERABILITY": { - "Critical": 10, - "High": 3, - "Medium": 20, - "Low": 30, - }, - "COMPLIANCE": {"Critical": 4}, - }, - "CHECKOV": { - "VERSION": "2.3.296", - "USE_EXTERNAL_CHECKS_GIT": "False", - "EXTERNAL_CHECKS_GIT": "rules", - "EXTERNAL_GIT_SSH_HOST": "github", - "EXTERNAL_GIT_PUBLIC_KEY_FINGERPRINT": "fingerprint", - "USE_EXTERNAL_CHECKS_DIR": "True", - "EXTERNAL_DIR_OWNER": "test", - "EXTERNAL_DIR_REPOSITORY": "repository", - "EXTERNAL_DIR_ASSET_NAME": "rules", - "RULES": "", - }, - } - mock_secret_tool = { - "github_token": "mock_github_token", - "repository_ssh_host": "repository_ssh_host", - } - - # Configurar el valor simulado de retorno para ciertos métodos - mock_github_api_instance = MagicMock() - mock_github_api.return_value = mock_github_api_instance +def test_retryable_install_package(checkov_tool): + subprocess_mock = MagicMock() + subprocess_mock.run.return_value.returncode = 1 - # Llamar al método que se está probando - result = self.checkov_tool.configurate_external_checks( - json_data, mock_secret_tool - ) - - # Verificar que el resultado es el esperado - self.assertIsNone(result) + with patch("subprocess.run", return_value=subprocess_mock) as mock_run: + response = checkov_tool.retryable_install_package("checkov", "2.3.96") - def test_configurate_external_checks_secret_tool_None(self): - # Llamar al método que se está probando - result = self.checkov_tool.configurate_external_checks(None, None) + mock_run.assert_called() + assert response is False - # Verificar que el resultado es el esperado - self.assertIsNone(result) +def test_execute(checkov_tool): + checkov_config = MagicMock() + checkov_config.path_config_file = "/path/to/config/" + checkov_config.config_file_name = "checkov_config" - @mock.patch( - "devsecops_engine_tools.engine_utilities.github.infrastructure.github_api.GithubApi.download_latest_release_assets", - autospec=True, - ) - def test_configurate_external_checks_error(self, mock_github_api): - # Configurar valores simulados - json_data = { - "SEARCH_PATTERN": ["AW", "NU"], - "IGNORE_SEARCH_PATTERN": [ - "test", - ], - "MESSAGE_INFO_ENGINE_IAC": "message test", - "EXCLUSIONS_PATH": "Exclusions.json", - "UPDATE_SERVICE_WITH_FILE_NAME_CFT": "false", - "THRESHOLD": { - "VULNERABILITY": { - "Critical": 10, - "High": 3, - "Medium": 20, - "Low": 30, - }, - "COMPLIANCE": {"Critical": 4}, - }, - "CHECKOV": { - "VERSION": "2.3.296", - "USE_EXTERNAL_CHECKS_GIT": "False", - "EXTERNAL_CHECKS_GIT": "rules", - "EXTERNAL_GIT_SSH_HOST": "github", - "EXTERNAL_GIT_PUBLIC_KEY_FINGERPRINT": "fingerprint", - "USE_EXTERNAL_CHECKS_DIR": "True", - "EXTERNAL_DIR_OWNER": "test", - "EXTERNAL_DIR_REPOSITORY": "repository", - "RULES": "", - }, - } - mock_secret_tool = { - "github_token": "mock_github_token", - "repository_ssh_host": "repository_ssh_host", - } + subprocess_mock = MagicMock() + subprocess_mock.run.return_value.stdout = "Output" + subprocess_mock.run.return_value.stderr = "Error" - # Configurar el valor simulado de retorno para ciertos métodos - mock_github_api.side_effect = Exception("Simulated error") + with patch("subprocess.run", return_value=subprocess_mock) as mock_run: + checkov_tool.execute(checkov_config) - # Llamar al método que se está probando - result = self.checkov_tool.configurate_external_checks( - json_data, mock_secret_tool + mock_run.assert_called_once_with( + "checkov --config-file /path/to/config/checkov_configcheckov_config.yaml", + capture_output=True, + text=True, + shell=True, + env=dict(os.environ), ) - # Verificar que el resultado es el esperado - self.assertIsNone(result) - - def test_retryable_install_package(self): - subprocess_mock = MagicMock() - subprocess_mock.run.return_value.returncode = 1 - - with mock.patch("subprocess.run", return_value=subprocess_mock) as mock_run: - response = self.checkov_tool.retryable_install_package("checkov", "2.3.96") - - mock_run.assert_called() - self.assertEqual(response, False) - - def test_execute(self): - checkov_config = MagicMock() - checkov_config.path_config_file = "/path/to/config/" - checkov_config.config_file_name = "checkov_config" - - subprocess_mock = MagicMock() - subprocess_mock.run.return_value.stdout = "Output" - subprocess_mock.run.return_value.stderr = "Error" - - with mock.patch("subprocess.run", return_value=subprocess_mock) as mock_run: - self.checkov_tool.execute(checkov_config) - - mock_run.assert_called_once_with( - "checkov --config-file /path/to/config/checkov_configcheckov_config.yaml", - capture_output=True, - text=True, - shell=True, - env=dict(os.environ), - ) - - @mock.patch( - "devsecops_engine_tools.engine_sast.engine_iac.src.infrastructure.driven_adapters.checkov.checkov_tool.CheckovTool.execute", - autospec=True, - ) - def test_async_scan(self, mock_checkov_tool): - checkov_config = MagicMock() - checkov_config.path_config_file = "/path/to/config/" - checkov_config.config_file_name = "checkov_config" - - output_queue = Queue() - - mock_checkov_tool.return_value = '{"key": "value"}' - - self.checkov_tool.async_scan(output_queue, checkov_config) - - self.assertEqual(output_queue.get(), [{"key": "value"}]) - - def test_scan_folders(self): - folders_to_scan = ["/path/to/folder"] - config_tool = MagicMock() - config_tool = { - "CHECKOV": { - "USE_EXTERNAL_CHECKS_GIT": "False", - "USE_EXTERNAL_CHECKS_DIR": "True", - "EXTERNAL_DIR_OWNER": "test", - "EXTERNAL_DIR_REPOSITORY": "repository", - "EXTERNAL_CHECKS_GIT": "rules", - "RULES": { - "RULES_DOCKER": {"rule1": {"environment": {"dev": True}}}, - "RULES_K8S": {"rule2": {"environment": {"prod": True}}}, - } +@patch( + "devsecops_engine_tools.engine_sast.engine_iac.src.infrastructure.driven_adapters.checkov.checkov_tool.CheckovTool.execute", + autospec=True, +) +def test_async_scan(mock_checkov_tool, checkov_tool): + checkov_config = MagicMock() + checkov_config.path_config_file = "/path/to/config/" + checkov_config.config_file_name = "checkov_config" + + output_queue = Queue() + + mock_checkov_tool.return_value = '{"key": "value"}' + + checkov_tool.async_scan(output_queue, checkov_config) + + assert output_queue.get() == [{"key": "value"}] + +def test_scan_folders(checkov_tool): + folders_to_scan = ["/path/to/folder"] + config_tool = { + "CHECKOV": { + "USE_EXTERNAL_CHECKS_GIT": "False", + "USE_EXTERNAL_CHECKS_DIR": "True", + "EXTERNAL_DIR_OWNER": "test", + "EXTERNAL_DIR_REPOSITORY": "repository", + "EXTERNAL_CHECKS_GIT": "rules", + "RULES": { + "RULES_DOCKER": {"rule1": {"environment": {"dev": True}}}, + "RULES_K8S": {"rule2": {"environment": {"prod": True}}}, } } - agent_env = MagicMock() - environment = "dev" - - output_queue = Queue() - output_queue.put([{"key": "value"}]) - - with mock.patch.object( - self.checkov_tool, "async_scan", side_effect=output_queue.put - ): - result_scans, rules_run = self.checkov_tool.scan_folders( - folders_to_scan, config_tool, agent_env, environment, "eks" - ) + } + agent_env = MagicMock() + environment = "dev" + + output_queue = Queue() + output_queue.put([{"key": "value"}]) + + with patch.object( + checkov_tool, "async_scan", side_effect=output_queue.put + ): + result_scans, rules_run = checkov_tool.scan_folders( + folders_to_scan, config_tool, agent_env, environment, "eks" + ) - self.assertEqual(result_scans, []) + assert result_scans == [] - def test_run_tool(self): - config_tool = MagicMock() - folders_to_scan = ["/path/to/folder"] - environment = "dev" - platform = "eks" - secret_tool = MagicMock() +def test_run_tool(checkov_tool): + config_tool = MagicMock() + folders_to_scan = ["/path/to/folder"] + environment = "dev" + platform = "eks" + secret_tool = MagicMock() - self.checkov_tool.configurate_external_checks = MagicMock( - return_value="agent_env" - ) - self.checkov_tool.scan_folders = MagicMock(return_value=[{"key": "value"}, []]) - self.checkov_tool.TOOL_CHECKOV = "CHECKOV" + checkov_tool.configurate_external_checks = MagicMock( + return_value="agent_env" + ) + checkov_tool.scan_folders = MagicMock(return_value=[{"key": "value"}, []]) + checkov_tool.TOOL_CHECKOV = "CHECKOV" - findings_list, file_from_tool = self.checkov_tool.run_tool( - config_tool, folders_to_scan, environment, platform, secret_tool, secret_external_checks="github:token" - ) + findings_list, file_from_tool = checkov_tool.run_tool( + config_tool, folders_to_scan, environment, platform, secret_tool, secret_external_checks="github:token" + ) - self.assertEqual(findings_list, []) + assert findings_list == [] diff --git a/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/helpers/test_file_generator_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/helpers/test_file_generator_tool.py index c5b9e1b82..4fe6ab943 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/helpers/test_file_generator_tool.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_iac/test/infrastructure/helpers/test_file_generator_tool.py @@ -126,7 +126,7 @@ def test_generate_file_from_tool(): }, } - absolute_path = generate_file_from_tool("CHECKOV", results_scan_list, rules_doc) + absolute_path = generate_file_from_tool("CHECKOV", results_scan_list, rules_doc, "", "") with open(absolute_path, "r") as file: data = file.read() @@ -232,7 +232,7 @@ def test_generate_file_from_tool_exception(): } ] - absolute_path = generate_file_from_tool("CHECKOV", results_scan_list, None) + absolute_path = generate_file_from_tool("CHECKOV", results_scan_list, None, "", "") assert absolute_path == None \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/applications/runner_secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/applications/runner_secret_scan.py index ed02abad1..b4985318f 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/applications/runner_secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/applications/runner_secret_scan.py @@ -7,6 +7,12 @@ from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.trufflehog.trufflehog_deserealizator import ( SecretScanDeserealizator ) +from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_tool import ( + GitleaksTool + ) +from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_deserealizator import ( + GitleaksDeserealizator + ) from devsecops_engine_tools.engine_utilities.git_cli.infrastructure.git_run import ( GitRun ) @@ -19,6 +25,10 @@ def runner_secret_scan(dict_args, tool, devops_platform_gateway, secret_tool): if (tool == "TRUFFLEHOG"): tool_gateway = TrufflehogRun() tool_deserealizator = SecretScanDeserealizator() + elif (tool == "GITLEAKS"): + tool_gateway = GitleaksTool() + tool_deserealizator = GitleaksDeserealizator() + return engine_secret_scan( devops_platform_gateway = devops_platform_gateway, tool_gateway = tool_gateway, diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py deleted file mode 100644 index 4bfe25dac..000000000 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py +++ /dev/null @@ -1,14 +0,0 @@ -from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold - -class DeserializeConfigTool: - def __init__(self, json_data, tool): - self.ignore_search_pattern = json_data["IGNORE_SEARCH_PATTERN"] - self.message_info_engine_secret = json_data["MESSAGE_INFO_ENGINE_SECRET"] - self.level_compliance = Threshold(json_data['THRESHOLD']) - self.scope_pipeline = '' - self.exclude_path = json_data[tool]["EXCLUDE_PATH"] - self.number_threads = json_data[tool]["NUMBER_THREADS"] - self.target_branches = json_data["TARGET_BRANCHES"] - self.enable_custom_rules = json_data[tool]["ENABLE_CUSTOM_RULES"] - self.external_dir_owner = json_data[tool]["EXTERNAL_DIR_OWNER"] - self.external_dir_repo = json_data[tool]["EXTERNAL_DIR_REPOSITORY"] diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py index f63ba5507..373020514 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py @@ -1,9 +1,8 @@ from abc import ABCMeta, abstractmethod -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import DeserializeConfigTool class ToolGateway(metaclass=ABCMeta): @abstractmethod - def install_tool(self, agent_os: str, agent_temp_dir:str) -> any: + def install_tool(self, agent_os: str, agent_temp_dir:str, version: str) -> any: "install tool" @abstractmethod def run_tool_secret_scan(self, @@ -11,7 +10,9 @@ def run_tool_secret_scan(self, agent_os: str, agent_work_folder: str, repository_name: str, - config_tool: DeserializeConfigTool, + config_tool, secret_tool, - secret_external_checks) -> str: + secret_external_checks, + agent_tem_dir:str, + tool) -> str: "run tool secret scan" \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py index 8fabe8ceb..644cb407b 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py @@ -1,7 +1,5 @@ -from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool, -) +import re + from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway import ( ToolGateway, ) @@ -28,56 +26,69 @@ def __init__( self.tool_deserialize = tool_deserialize self.git_gateway = git_gateway - def process(self, skip_tool, config_tool, secret_tool, dict_args): + def process(self, skip_tool, config_tool, secret_tool, dict_args, tool): + tool = str(tool).lower() finding_list = [] file_path_findings = "" secret_external_checks=dict_args["token_external_checks"] + files_to_scan = None if dict_args["folder_path"] is None else [dict_args["folder_path"]] if skip_tool == False: - self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory")) - files_pullrequest = self.git_gateway.get_files_pull_request( - self.devops_platform_gateway.get_variable("path_directory"), - self.devops_platform_gateway.get_variable("target_branch"), - config_tool.target_branches, - self.devops_platform_gateway.get_variable("source_branch"), - self.devops_platform_gateway.get_variable("access_token"), - self.devops_platform_gateway.get_variable("organization"), - self.devops_platform_gateway.get_variable("project_name"), - self.devops_platform_gateway.get_variable("repository"), - self.devops_platform_gateway.get_variable("repository_provider")) + self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"), config_tool[tool]["VERSION"]) + if files_to_scan is None: + files_to_scan = self.git_gateway.get_files_pull_request( + self.devops_platform_gateway.get_variable("path_directory"), + self.devops_platform_gateway.get_variable("target_branch"), + config_tool["TARGET_BRANCHES"], + self.devops_platform_gateway.get_variable("source_branch"), + self.devops_platform_gateway.get_variable("access_token"), + self.devops_platform_gateway.get_variable("organization"), + self.devops_platform_gateway.get_variable("project_name"), + self.devops_platform_gateway.get_variable("repository"), + self.devops_platform_gateway.get_variable("repository_provider")) findings, file_path_findings = self.tool_gateway.run_tool_secret_scan( - files_pullrequest, + files_to_scan, self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("path_directory"), self.devops_platform_gateway.get_variable("repository"), config_tool, secret_tool, - secret_external_checks) + secret_external_checks, + self.devops_platform_gateway.get_variable("temp_directory"), + tool) finding_list = self.tool_deserialize.get_list_vulnerability( findings, self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("path_directory") ) + else: + print("Tool skipped by DevSecOps policy") + dict_args["send_metrics"] = "false" return finding_list, file_path_findings def complete_config_tool(self, dict_args, tool): tool = str(tool).lower() init_config_tool = self.devops_platform_gateway.get_remote_config( - dict_args["remote_config_repo"], "engine_sast/engine_secret/ConfigTool.json" + dict_args["remote_config_repo"], "engine_sast/engine_secret/ConfigTool.json", dict_args["remote_config_branch"] ) - config_tool = DeserializeConfigTool(json_data=init_config_tool, tool=tool) - config_tool.scope_pipeline = self.devops_platform_gateway.get_variable("pipeline_name") - return config_tool + init_config_tool['SCOPE_PIPELINE'] = self.devops_platform_gateway.get_variable("pipeline_name") + + skip_tool = bool(re.match(init_config_tool["IGNORE_SEARCH_PATTERN"], init_config_tool["SCOPE_PIPELINE"], re.IGNORECASE)) - def skip_from_exclusion(self, exclusions): + return init_config_tool, skip_tool + + def skip_from_exclusion(self, exclusions, skip_tool_isp): """ Handle skip tool. Return: bool: True -> skip tool, False -> not skip tool. """ - pipeline_name = self.devops_platform_gateway.get_variable("pipeline_name") - if (pipeline_name in exclusions) and ( - exclusions[pipeline_name].get("SKIP_TOOL", 0) - ): + if(skip_tool_isp): return True else: - return False \ No newline at end of file + pipeline_name = self.devops_platform_gateway.get_variable("pipeline_name") + if (pipeline_name in exclusions) and ( + exclusions[pipeline_name].get("SKIP_TOOL", 0) + ): + return True + else: + return False \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py index 927304d2b..a1f034c3b 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py @@ -2,14 +2,18 @@ from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import ( DevopsPlatformGateway, ) -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool - ) from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions - +from devsecops_engine_tools.engine_utilities.utils.utils import Utils +from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold class SetInputCore: - def __init__(self, tool_remote: DevopsPlatformGateway, dict_args, tool, config_tool: DeserializeConfigTool): + def __init__( + self, + tool_remote: DevopsPlatformGateway, + dict_args, + tool, + config_tool, + ): self.tool_remote = tool_remote self.dict_args = dict_args self.tool = tool @@ -22,7 +26,9 @@ def get_remote_config(self, file_path): Returns: dict: Remote configuration. """ - return self.tool_remote.get_remote_config(self.dict_args["remote_config_repo"], file_path) + return self.tool_remote.get_remote_config( + self.dict_args["remote_config_repo"], file_path, self.dict_args["remote_config_branch"] + ) def get_variable(self, variable): """ @@ -60,15 +66,23 @@ def set_input_core(self, finding_list): Returns: dict: Input core. """ + exclusions_config = self.get_remote_config( + "engine_sast/engine_secret/Exclusions.json" + ) return InputCore( totalized_exclusions=self.get_exclusions( - self.get_remote_config("engine_sast/engine_secret/Exclusions.json"), + exclusions_config, self.get_variable("pipeline_name"), self.tool, ), - threshold_defined=self.config_tool.level_compliance, + threshold_defined=Utils.update_threshold( + self, + Threshold(self.config_tool['THRESHOLD']), + exclusions_config, + self.config_tool["SCOPE_PIPELINE"], + ), path_file_results=finding_list, - custom_message_break_build=self.config_tool.message_info_engine_secret, - scope_pipeline=self.config_tool.scope_pipeline, - stage_pipeline=self.tool_remote.get_variable("stage").capitalize() + custom_message_break_build=self.config_tool["MESSAGE_INFO_ENGINE_SECRET"], + scope_pipeline=self.config_tool["SCOPE_PIPELINE"], + stage_pipeline=self.tool_remote.get_variable("stage").capitalize(), ) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/__init__.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/gitleaks_deserealizator.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/gitleaks_deserealizator.py new file mode 100644 index 000000000..aec0fa2c6 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/gitleaks_deserealizator.py @@ -0,0 +1,36 @@ +from datetime import datetime +from dataclasses import dataclass +from typing import List +from devsecops_engine_tools.engine_core.src.domain.model.finding import Finding, Category +from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.gateway_deserealizator import ( + DeseralizatorGateway +) + +@dataclass +class GitleaksDeserealizator(DeseralizatorGateway): + + def get_list_vulnerability(self, results_scan_list: List[dict], path_directory: str, os: str) -> List[Finding]: + list_open_vulnerabilities = [] + current_date=datetime.now().strftime("%d%m%Y") + + for result in results_scan_list: + vulnerability_open = Finding( + id=result.get("RuleID", "SECRET_SCANNING"), + cvss=None, + where=self.get_where_correctly(result, path_directory), + description=result.get("Description", "No description available"), + severity="critical", + identification_date=current_date, + published_date_cve=None, + module="engine_secret", + category=Category.VULNERABILITY, + requirements="", + tool="Gitleaks", + ) + list_open_vulnerabilities.append(vulnerability_open) + return list_open_vulnerabilities + + def get_where_correctly(self, result: dict, path_directory=""): + path = result.get("File", "").replace(path_directory, "") + hidden_secret = str(result.get("Secret"))[:3] + '*' * 9 + str(result.get("Secret"))[-3:] + return f"{path}, Secret: {hidden_secret}" \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/gitleaks_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/gitleaks_tool.py new file mode 100644 index 000000000..d44c2c5d0 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/gitleaks_tool.py @@ -0,0 +1,150 @@ +import json +import os +import re +import subprocess +import requests +from concurrent.futures import ThreadPoolExecutor, as_completed +from devsecops_engine_tools.engine_utilities.utils.utils import Utils +from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway import ( + ToolGateway, +) +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities import settings +from devsecops_engine_tools.engine_utilities.utils.utils import Utils + +logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() + +class GitleaksTool(ToolGateway): + _COMMAND = None + + def install_tool(self, agent_os, agent_temp_dir, tool_version) -> any: + is_windows_os = re.search(r"Windows", agent_os) + is_linux_os = re.search(r"Linux", agent_os) + + if is_windows_os: + file_extension = "windows_x64.zip" + elif is_linux_os: + file_extension = "linux_x64.tar.gz" + else: + file_extension = "darwin_x64.tar.gz" + + command = f"{agent_temp_dir}{os.sep}gitleaks" + command = f"{command}.exe" if is_windows_os else command + + self._COMMAND = command + result = subprocess.run(f"{command} --version", capture_output=True, shell=True, text=True) + is_tool_installed = re.search(fr"{tool_version}", result.stdout.strip()) + + if is_tool_installed: return + + try: + url = f"https://github.com/gitleaks/gitleaks/releases/download/v{tool_version}/gitleaks_{tool_version}_{file_extension}" + response = requests.get(url, allow_redirects=True) + compressed_name = os.path.join( + agent_temp_dir, f"gitleaks_{tool_version}_{file_extension}" + ) + with open(compressed_name, "wb") as f: + f.write(response.content) + + if is_windows_os: + Utils().unzip_file(compressed_name, agent_temp_dir) + else: + Utils().extract_targz_file(compressed_name, agent_temp_dir) + + except Exception as ex: + logger.error(f"An error ocurred downloading Gitleaks: {ex}") + + def _extract_json_data(self, file_path): + if os.path.exists(file_path): + with open(file_path, 'r', encoding='utf-8') as f: + return json.load(f) + else: + print(f"File {file_path} does not exist") + return [] + + def _create_report(self, output_file, combined_data): + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(combined_data, f, ensure_ascii=False, indent=4) + + def _check_path(self, path, excluded_paths): + parts = path.split(os.sep) + for part in parts: + if part in excluded_paths: return True + return False + + def _add_flags(self, config_tool, tool, agent_work_folder): + flags = [] + if not config_tool[tool]["ALLOW_IGNORE_LEAKS"]: + flags.append("--ignore-gitleaks-allow") + + if config_tool[tool]["ENABLE_CUSTOM_RULES"]: + flags.extend(["--config", f"{agent_work_folder}{os.sep}rules{os.sep}gitleaks{os.sep}gitleaks.toml"]) + + return flags + + def run_tool_secret_scan( + self, + files, + agent_os, + agent_work_folder, + repository_name, + config_tool, + secret_tool, # For external checks + secret_external_checks, # For external checks + agent_temp_dir, + tool + ): + command = [self._COMMAND, "dir"] + finding_path = os.path.join(agent_work_folder, "gitleaks_report.json") + excluded_paths = config_tool[tool]["EXCLUDE_PATH"] + + if config_tool[tool]["ENABLE_CUSTOM_RULES"]: + Utils().configurate_external_checks(tool, config_tool, secret_tool, secret_external_checks, agent_work_folder) + + try: + findings = [] + flags = self._add_flags(config_tool, tool, agent_work_folder) + if len(files) > 1: + with ThreadPoolExecutor(max_workers=config_tool[tool]["NUMBER_THREADS"]) as executor: + futures = [] + + for pull_file in files: + if self._check_path(pull_file, excluded_paths): continue + + aux_finding_path = os.path.join( + agent_work_folder, f"gitleaks_aux_report_{pull_file.replace(os.sep, '_')}.json" + ) + + command_aux = command.copy() + command_aux.extend([ + os.path.join(agent_work_folder, repository_name, pull_file), + "--report-path", aux_finding_path + ]) + command_aux.extend(flags) + + futures.append(executor.submit(self._run_subprocess_command, command_aux, aux_finding_path)) + + for future in as_completed(futures): + result = future.result() + findings.extend(result) + + self._create_report(finding_path, findings) + else: + command.extend([files[0], "--report-path", finding_path]) + command.extend(flags) + + subprocess.run(command, capture_output=True, text=True) + findings = self._extract_json_data(finding_path) + + return findings, finding_path + + except Exception as e: + logger.error(f"Error executing gitleaks scan: {e}") + + def _run_subprocess_command(self, command_aux, aux_finding_path): + try: + subprocess.run(command_aux, capture_output=True, text=True) + return self._extract_json_data(aux_finding_path) + except Exception as e: + logger.error(f"Error executing gitleaks on {command_aux}: {e}") + return [] \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_deserealizator.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_deserealizator.py old mode 100644 new mode 100755 index 7a9c44503..0ebfa8664 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_deserealizator.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_deserealizator.py @@ -10,15 +10,26 @@ class SecretScanDeserealizator(DeseralizatorGateway): def get_list_vulnerability(self, results_scan_list: List[dict], os, path_directory) -> List[Finding]: list_open_vulnerabilities = [] + current_date=datetime.now().strftime("%d%m%Y") + for result in results_scan_list: - where_text, raw = self.get_where_correctly(result, os, path_directory) + where_text, raw_data = self.get_where_correctly(result, os, path_directory) + rule_name = result.get("Id", {}) + + if "MISCONFIGURATION_SCANNING" in rule_name: + description = "Actuator misconfiguration can leak sensitive information" + where = f"{where_text}, Misconfiguration: {raw_data}" + else: + description = "Sensitive information in source code" + where = f"{where_text}, Secret: {raw_data}" + vulnerability_open = Finding( - id="SECRET_SCANNING", + id=result.get("Id", {}), cvss=None, - where=f"{where_text}, Secret: {raw}", - description="Sensitive information in source code", + where=where, + description=description, severity="critical", - identification_date=datetime.now().strftime("%d%m%Y"), + identification_date=current_date, published_date_cve=None, module="engine_secret", category=Category.VULNERABILITY, diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py old mode 100644 new mode 100755 index b78a29b5d..d385fc4c8 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py @@ -7,11 +7,10 @@ from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway import ( ToolGateway, ) -from devsecops_engine_tools.engine_utilities.github.infrastructure.github_api import ( - GithubApi, -) + from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings +from devsecops_engine_tools.engine_utilities.utils.utils import Utils logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() @@ -19,26 +18,35 @@ class TrufflehogRun(ToolGateway): - def install_tool(self, agent_os, agent_temp_dir) -> any: + def install_tool(self, agent_os, agent_temp_dir, tool_version) -> any: reg_exp_os = r"Windows" check_os = re.search(reg_exp_os, agent_os) + reg_exp_tool = fr"{tool_version}" if check_os: - self.run_install_win(agent_temp_dir) + command = f"{agent_temp_dir}/trufflehog.exe --version" + subprocess.run(command, shell=True) + result = subprocess.run(command, capture_output=True, shell=True) + output = result.stderr.strip() + check_tool = re.search(reg_exp_tool, output.decode("utf-8")) + if not check_tool: + self.run_install_win(agent_temp_dir, tool_version) + subprocess.run(command, shell=True) else: command = f"trufflehog --version" + subprocess.run(command, shell=True) result = subprocess.run(command, capture_output=True, shell=True) output = result.stderr.strip() - reg_exp = r"not found" - check_tool = re.search(reg_exp, output.decode("utf-8")) - if check_tool: - self.run_install() + check_tool = re.search(reg_exp_tool, output.decode("utf-8")) + if not check_tool: + self.run_install(tool_version) + subprocess.run(command, shell=True) - def run_install(self): - command = f"curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin" + def run_install(self, tool_version): + command = f"curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin v{tool_version}" subprocess.run(command, capture_output=True, shell=True) - def run_install_win(self, agent_temp_dir): - command_complete = f"powershell -Command [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; [Net.ServicePointManager]::SecurityProtocol; New-Item -Path {agent_temp_dir} -ItemType Directory -Force; Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh' -OutFile {agent_temp_dir}\install_trufflehog.sh; bash {agent_temp_dir}\install_trufflehog.sh -b C:/Trufflehog/bin; $env:Path += ';C:/Trufflehog/bin'; C:/Trufflehog/bin/trufflehog.exe --version" + def run_install_win(self, agent_temp_dir, tool_version): + command_complete = f"powershell -Command [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; [Net.ServicePointManager]::SecurityProtocol; New-Item -Path {agent_temp_dir} -ItemType Directory -Force; Invoke-WebRequest -Uri 'https://github.com/trufflesecurity/trufflehog/releases/download/v{tool_version}/trufflehog_{tool_version}_windows_amd64.tar.gz' -OutFile {agent_temp_dir}/trufflehog.tar.gz -UseBasicParsing; tar -xzf {agent_temp_dir}/trufflehog.tar.gz -C {agent_temp_dir}; Remove-Item {agent_temp_dir}/trufflehog.tar.gz; $env:Path += '; + {agent_temp_dir}'; & {agent_temp_dir}/trufflehog.exe --version" process = subprocess.Popen( command_complete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) @@ -52,29 +60,22 @@ def run_tool_secret_scan( repository_name, config_tool, secret_tool, - secret_external_checks + secret_external_checks, + agent_temp_dir, + tool ): trufflehog_command = "trufflehog" if "Windows" in agent_os: - trufflehog_command = "C:/Trufflehog/bin/trufflehog.exe" + trufflehog_command = f"{agent_temp_dir}/trufflehog.exe" with open(f"{agent_work_folder}/excludedPath.txt", "w") as file: - file.write("\n".join(config_tool.exclude_path)) + file.write("\n".join(config_tool[tool]["EXCLUDE_PATH"])) exclude_path = f"{agent_work_folder}/excludedPath.txt" - include_paths = self.config_include_path(files_commits, agent_work_folder) - enable_custom_rules = config_tool.enable_custom_rules.lower() - secret = None - - if secret_tool is not None: - secret = secret_tool["github_token"] if "github_token" in secret_tool else None - elif secret_external_checks is not None: - secret = secret_external_checks.split("github:")[1] if "github" in secret_external_checks else None + include_paths = self.config_include_path(files_commits, agent_work_folder, agent_os) + enable_custom_rules = config_tool[tool]["ENABLE_CUSTOM_RULES"] + if enable_custom_rules: + Utils().configurate_external_checks(tool, config_tool, secret_tool, secret_external_checks, agent_work_folder) - if enable_custom_rules == "true" and secret is not None: - self.configurate_external_checks(config_tool, secret) - else: #In case that remote config from tool is enable but in the args dont send any type of secrets. So dont modified command - enable_custom_rules = "false" - - with concurrent.futures.ThreadPoolExecutor(max_workers=config_tool.number_threads) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=config_tool[tool]["NUMBER_THREADS"]) as executor: results = executor.map( self.run_trufflehog, [trufflehog_command] * len(include_paths), @@ -83,11 +84,12 @@ def run_tool_secret_scan( include_paths, [repository_name] * len(include_paths), [enable_custom_rules] * len(include_paths), + [agent_os] * len(include_paths) ) - findings, file_findings = self.create_file(self.decode_output(results), agent_work_folder) + findings, file_findings = self.create_file(self.decode_output(results), agent_work_folder, config_tool, tool) return findings, file_findings - def config_include_path(self, files, agent_work_folder): + def config_include_path(self, files, agent_work_folder, agent_os): chunks = [] if len(files) != 0: chunk_size = (len(files) + 3) // 4 @@ -102,6 +104,8 @@ def config_include_path(self, files, agent_work_folder): include_paths.append(file_path) with open(file_path, "w") as file: for file_pr_path in chunk: + if "Windows" in agent_os: + file_pr_path = str(file_pr_path).replace("/","\\\\") file.write(f"{file_pr_path.strip()}\n") return include_paths @@ -112,14 +116,17 @@ def run_trufflehog( exclude_path, include_path, repository_name, - enable_custom_rules + enable_custom_rules, + agent_os ): - command = f"{trufflehog_command} filesystem {agent_work_folder + '/' + repository_name} --include-paths {include_path} --exclude-paths {exclude_path} --no-verification --json" - - if enable_custom_rules == "true": - command = command.replace("--no-verification --json", "--config /tmp/rules/trufflehog/custom-rules.yaml --no-verification --json") + command = f"{trufflehog_command} filesystem {agent_work_folder + '/' + repository_name} --include-paths {include_path} --exclude-paths {exclude_path} --no-verification --no-update --json" - result = subprocess.run(command, capture_output=True, shell=True, text=True) + if enable_custom_rules: + command = command.replace("--no-verification --no-update --json", f"--config {agent_work_folder}//rules//trufflehog//custom-rules.yaml --no-verification --no-update --json" if "Windows" in agent_os else + "/tmp/rules/trufflehog/custom-rules.yaml --no-verification --no-update --json" if "Linux" in agent_os else + "--no-verification --no-update --json") + + result = subprocess.run(command, capture_output=True, shell=True, text=True, encoding='utf-8') return result.stdout.strip() def decode_output(self, results): @@ -132,7 +139,7 @@ def decode_output(self, results): result.append(json_obj) return result - def create_file(self, findings, agent_work_folder): + def create_file(self, findings, agent_work_folder, config_tool, tool): file_findings = os.path.join(agent_work_folder, "secret_scan_result.json") with open(file_findings, "w") as file: for find in findings: @@ -140,17 +147,9 @@ def create_file(self, findings, agent_work_folder): original_where = original_where.replace("\\", "/") where_text = original_where.replace(agent_work_folder, "") find["SourceMetadata"]["Data"]["Filesystem"]["file"] = where_text + find["Id"] = "MISCONFIGURATION_SCANNING" if "exposure" in find["Raw"] else "SECRET_SCANNING" + find["References"] = config_tool[tool]["RULES"][find["Id"]]["References"] if "SECRET_SCANNING" not in find["Id"] else "N.A" + find["Mitigation"] = config_tool[tool]["RULES"][find["Id"]]["Mitigation"] if "SECRET_SCANNING" not in find["Id"] else "N.A" json_str = json.dumps(find) file.write(json_str + '\n') - return findings, file_findings - - def configurate_external_checks(self, config_tool, secret): - try: - github_api = GithubApi(secret) - github_api.download_latest_release_assets( - config_tool.external_dir_owner, - config_tool.external_dir_repo, - "/tmp", - ) - except Exception as ex: - logger.error(f"An error ocurred download external checks {ex}") \ No newline at end of file + return findings, file_findings \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py index 2f04c28bd..25c1b590e 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py @@ -6,11 +6,11 @@ def engine_secret_scan(devops_platform_gateway, tool_gateway, dict_args, tool, tool_deserealizator, git_gateway, secret_tool): exclusions = devops_platform_gateway.get_remote_config( - dict_args["remote_config_repo"], "engine_sast/engine_secret/Exclusions.json" + dict_args["remote_config_repo"], "engine_sast/engine_secret/Exclusions.json", dict_args["remote_config_branch"] ) secret_scan = SecretScan(tool_gateway, devops_platform_gateway, tool_deserealizator, git_gateway) - config_tool = secret_scan.complete_config_tool(dict_args, tool) - skip_tool = secret_scan.skip_from_exclusion(exclusions) - finding_list, file_path_findings = secret_scan.process(skip_tool, config_tool, secret_tool, dict_args) + config_tool, skip_tool_isp = secret_scan.complete_config_tool(dict_args, tool) + skip_tool = secret_scan.skip_from_exclusion(exclusions, skip_tool_isp) + finding_list, file_path_findings = secret_scan.process(skip_tool, config_tool, secret_tool, dict_args, tool) input_core = SetInputCore(devops_platform_gateway, dict_args, tool, config_tool) return finding_list, input_core.set_input_core(file_path_findings) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/applications/test_runner_secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/applications/test_runner_secret_scan.py index f2b2e1199..0e9079325 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/applications/test_runner_secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/applications/test_runner_secret_scan.py @@ -39,7 +39,7 @@ def test_runner_secret_scan(mock_entry_point_tool): def test_runner_secret_scan_exception(mock_entry_point_tool): # Arrange dict_args = {'arg1': 'value1', 'arg2': 'value2'} - tool = 'TRUFFLEHOG' + tool = 'GITLEAKS' secret_tool = "secret" devops_platform_gateway = None diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py old mode 100644 new mode 100755 index 1fe98639d..39e5eb618 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py @@ -1,20 +1,14 @@ import unittest from unittest.mock import patch -from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore from devsecops_engine_tools.engine_sast.engine_secret.src.domain.usecases.secret_scan import ( SecretScan, ) -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool, -) class TestSecretScan(unittest.TestCase): def setUp(self) -> None: global json_config json_config = { - "IGNORE_SEARCH_PATTERN": [ - "test" - ], + "IGNORE_SEARCH_PATTERN": "(.*test.*)", "MESSAGE_INFO_ENGINE_SECRET": "If you have doubts, visit url", "THRESHOLD": { "VULNERABILITY": { @@ -29,11 +23,19 @@ def setUp(self) -> None: }, "TARGET_BRANCHES": ["trunk", "develop"], "trufflehog": { + "VERSION": "1.2.3", "EXCLUDE_PATH": [".git", "node_modules", "target", "build", "build.gradle", "twistcli-scan", ".svg", ".drawio"], "NUMBER_THREADS": 4, "ENABLE_CUSTOM_RULES" : "True", "EXTERNAL_DIR_OWNER": "ExternalOrg", - "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks" + "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks", + "APP_ID_GITHUB":"123123", + "INSTALLATION_ID_GITHUB":"234234", + "RULES": { + "MISSCONFIGURATION_SCANNING" : { + "References" : "https://link.reference.com" + } + } } } @@ -57,7 +59,8 @@ def test_process( mock_git_gateway_instance = mock_git_gateway.return_value mock_dict_args = { "remote_config_repo": "example_repo", - "folder_path": ".", + "remote_config_branch": "", + "folder_path": None, "environment": "test", "platform": "local", "token_external_checks": "fake_github_token", @@ -76,7 +79,6 @@ def test_process( "vulnerability_data" ] - obj_config_tool = DeserializeConfigTool(json_config, 'trufflehog') mock_devops_gateway_instance.get_remote_config.return_value = json_config mock_devops_gateway_instance.get_variable.return_value = "example_pipeline" mock_tool_gateway_instance.run_tool_secret_scan.return_value = ( @@ -84,7 +86,7 @@ def test_process( ) finding_list, file_path_findings = secret_scan.process( - False, obj_config_tool, secret_tool, mock_dict_args + False, json_config, secret_tool, mock_dict_args, "trufflehog" ) self.assertEqual(finding_list, ["vulnerability_data"]) @@ -112,6 +114,7 @@ def test_process_empty( mock_git_gateway_instance = mock_git_gateway.return_value mock_dict_args = { "remote_config_repo": "example_repo", + "remote_config_branch": "", "folder_path": ".", "environment": "test", "platform": "local", @@ -129,13 +132,12 @@ def test_process_empty( mock_deserialize_gateway_instance.get_list_vulnerability.return_value = [] - obj_config_tool = DeserializeConfigTool(json_config, 'trufflehog') mock_devops_gateway_instance.get_remote_config.return_value = json_config mock_devops_gateway_instance.get_variable.return_value = "example_pipeline" mock_tool_gateway_instance.run_tool_secret_scan.return_value = "", "" finding_list, file_path_findings = secret_scan.process( - False, obj_config_tool, secret_tool, mock_dict_args + False, json_config, secret_tool, mock_dict_args, "trufflehog" ) self.assertEqual(finding_list, []) @@ -165,13 +167,44 @@ def test_skip_tool_true(self, mock_tool_gateway, mock_devops_gateway, mock_deser mock_deserialize_gateway_instance, mock_git_gateway_instance ) + skip_tool_isp = False + mock_devops_gateway_instance = mock_devops_gateway.return_value + mock_devops_gateway_instance.get_variable.return_value = "test_pipeline" + exclusions = { + "test_pipeline": {"SKIP_TOOL": 1} + } + result = secret_scan.skip_from_exclusion(exclusions, skip_tool_isp) + self.assertTrue(result) + @patch('devsecops_engine_tools.engine_utilities.git_cli.model.gateway.git_gateway.GitGateway') + @patch( + "devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway.DevopsPlatformGateway" + ) + @patch( + "devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.gateway_deserealizator.DeseralizatorGateway" + ) + @patch( + "devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway.ToolGateway" + ) + def test_skip_tool_true_isp(self, mock_tool_gateway, mock_devops_gateway, mock_deserialize_gateway, mock_git_gateway): + mock_tool_gateway_instance = mock_tool_gateway.return_value + mock_devops_gateway_instance = mock_devops_gateway.return_value + mock_deserialize_gateway_instance = mock_deserialize_gateway.return_value + mock_git_gateway_instance = mock_git_gateway.return_value + + secret_scan = SecretScan( + mock_tool_gateway_instance, + mock_devops_gateway_instance, + mock_deserialize_gateway_instance, + mock_git_gateway_instance + ) + skip_tool_isp = True mock_devops_gateway_instance = mock_devops_gateway.return_value mock_devops_gateway_instance.get_variable.return_value = "test_pipeline" exclusions = { "test_pipeline": {"SKIP_TOOL": 1} } - result = secret_scan.skip_from_exclusion(exclusions) + result = secret_scan.skip_from_exclusion(exclusions, skip_tool_isp) self.assertTrue(result) @patch('devsecops_engine_tools.engine_utilities.git_cli.model.gateway.git_gateway.GitGateway') @@ -196,13 +229,13 @@ def test_skip_tool_false(self, mock_tool_gateway, mock_devops_gateway, mock_dese mock_deserialize_gateway_instance, mock_git_gateway_instance ) - + skip_tool_isp = False mock_devops_gateway_instance = mock_devops_gateway.return_value mock_devops_gateway_instance.get_variable.return_value = "other_pipeline" exclusions = { "test_pipeline": {"SKIP_TOOL": 1} } - result = secret_scan.skip_from_exclusion(exclusions) + result = secret_scan.skip_from_exclusion(exclusions, skip_tool_isp) self.assertFalse(result) @patch('devsecops_engine_tools.engine_utilities.git_cli.model.gateway.git_gateway.GitGateway') @@ -234,11 +267,11 @@ def test_complete_config_tool( mock_devops_gateway_instance.get_remote_config.return_value = json_config mock_devops_gateway_instance.get_variable.return_value = "example_pipeline" - config_tool_instance = secret_scan.complete_config_tool( - {"remote_config_repo": "repository"}, "TRUFFLEHOG" + config_tool_instance, skip_tool_isp = secret_scan.complete_config_tool( + {"remote_config_repo": "repository", "remote_config_branch": ""}, "TRUFFLEHOG" ) - self.assertEqual(config_tool_instance.scope_pipeline, "example_pipeline") + self.assertEqual(config_tool_instance["SCOPE_PIPELINE"], "example_pipeline") if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py index daa4031aa..da7602fb9 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py @@ -1,16 +1,10 @@ import pytest -import json from unittest.mock import MagicMock, Mock -from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore -from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import ( DevopsPlatformGateway, ) from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions from devsecops_engine_tools.engine_sast.engine_secret.src.domain.usecases.set_input_core import SetInputCore -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool - ) @pytest.fixture diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/gitleaks/__init__.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/gitleaks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/gitleaks/test_gitleaks_deserealizator.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/gitleaks/test_gitleaks_deserealizator.py new file mode 100644 index 000000000..b251b19cd --- /dev/null +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/gitleaks/test_gitleaks_deserealizator.py @@ -0,0 +1,68 @@ +import unittest +from unittest.mock import patch +from devsecops_engine_tools.engine_core.src.domain.model.finding import Finding, Category +from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_deserealizator import ( + GitleaksDeserealizator +) + +class TestGitleaksDeserealizator(unittest.TestCase): + + def setUp(self): + self.deserealizator = GitleaksDeserealizator() + + @patch("devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_deserealizator.datetime") + def test_get_list_vulnerability(self, MockDatetime): + # Arrange + MockDatetime.now.return_value.strftime.return_value = "27012024" + + results_scan_list = [ + { + "RuleID": "GITLEAKS_RULE_1", + "Description": "Hardcoded secret found", + "File": "/path/to/repo/file1.txt", + "Secret": "ABCDEFG123456789" + }, + { + "RuleID": "GITLEAKS_RULE_2", + "Description": "API key detected", + "File": "/path/to/repo/file2.txt", + "Secret": "ABCDEFG123456789" + } + ] + os = "Linux" + path_directory = "/path/to/repo" + + # Act + vulnerabilities = self.deserealizator.get_list_vulnerability(results_scan_list, path_directory, os) + + # Assert + self.assertEqual(len(vulnerabilities), 2) + + vulnerability = vulnerabilities[0] + self.assertIsInstance(vulnerability, Finding) + self.assertEqual(vulnerability.id, "GITLEAKS_RULE_1") + self.assertEqual(vulnerability.description, "Hardcoded secret found") + self.assertEqual(vulnerability.severity, "critical") + self.assertEqual(vulnerability.identification_date, "27012024") + self.assertEqual(vulnerability.tool, "Gitleaks") + self.assertEqual(vulnerability.category, Category.VULNERABILITY) + self.assertEqual(vulnerability.where, "/file1.txt, Secret: ABC*********789") + + vulnerability2 = vulnerabilities[1] + self.assertEqual(vulnerability2.id, "GITLEAKS_RULE_2") + self.assertEqual(vulnerability2.description, "API key detected") + self.assertEqual(vulnerability2.where, "/file2.txt, Secret: ABC*********789") + + def test_get_where_correctly(self): + # Arrange + result = { + "File": "/path/to/repo/file1.txt", + "Secret": "ABCDEFG123456789" + } + path_directory = "/path/to/repo" + + # Act + where_correctly = self.deserealizator.get_where_correctly(result, path_directory) + + # Assert + self.assertEqual(where_correctly, "/file1.txt, Secret: ABC*********789") diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/gitleaks/test_gitleaks_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/gitleaks/test_gitleaks_tool.py new file mode 100644 index 000000000..786f1fb27 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/gitleaks/test_gitleaks_tool.py @@ -0,0 +1,210 @@ +import unittest +from unittest.mock import patch, MagicMock, mock_open, call +import os +from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_tool import ( + GitleaksTool +) + +class TestGitleaksTool(unittest.TestCase): + + def setUp(self): + # Arrange + self.tool = GitleaksTool() + self.agent_work_folder = "/path/to/work/folder" + self.agent_temp_dir = "/path/to/temp/dir" + self.repository_name = "test_repo" + self.config_tool = { + "gitleaks": { + "VERSION": "8.20.0", + "EXCLUDE_PATH": ["excluded_dir"], + "NUMBER_THREADS": 2, + "ALLOW_IGNORE_LEAKS": False, + "ENABLE_CUSTOM_RULES" : False + } + } + + @patch("subprocess.run") + @patch("re.search") + def test_install_tool_windows(self, mock_search, mock_run): + # Arrange + mock_search.side_effect = [True, None, None] + mock_run.return_value = MagicMock(stdout="command not found") + + with patch("requests.get") as mock_requests, patch("builtins.open", mock_open()) as mock_file, patch("devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_tool.Utils.unzip_file") as mock_unzip: + mock_requests.return_value.content = b"compressed_data" + + # Act + self.tool.install_tool("Windows_NT", self.agent_temp_dir, "8.0.0") + + # Assert + mock_requests.assert_called_once_with( + "https://github.com/gitleaks/gitleaks/releases/download/v8.0.0/gitleaks_8.0.0_windows_x64.zip", + allow_redirects=True + ) + mock_file.assert_called_once_with(f"{self.agent_temp_dir}/gitleaks_8.0.0_windows_x64.zip", "wb") + mock_unzip.assert_called_once() + + @patch("subprocess.run") + @patch("re.search") + def test_install_tool_linux(self, mock_search, mock_run): + # Arrange + mock_search.side_effect = [None, True, None] + mock_run.return_value = MagicMock(stdout="command not found") + + with patch("requests.get") as mock_requests, patch("builtins.open", mock_open()) as mock_file, patch("devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_tool.Utils.extract_targz_file") as mock_extract: + mock_requests.return_value.content = b"compressed_data" + + # Act + self.tool.install_tool("Linux", self.agent_temp_dir, "8.0.0") + + # Assert + mock_requests.assert_called_once_with( + "https://github.com/gitleaks/gitleaks/releases/download/v8.0.0/gitleaks_8.0.0_linux_x64.tar.gz", + allow_redirects=True + ) + mock_file.assert_called_once_with(f"{self.agent_temp_dir}/gitleaks_8.0.0_linux_x64.tar.gz", "wb") + mock_extract.assert_called_once() + + @patch("subprocess.run") + @patch("re.search") + def test_install_tool_darwin(self, mock_search, mock_run): + # Arrange + mock_search.side_effect = [None, None, None] + mock_run.return_value = MagicMock(stdout="command not found") + + with patch("requests.get") as mock_requests, patch("builtins.open", mock_open()) as mock_file, patch("devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_tool.Utils.extract_targz_file") as mock_extract: + mock_requests.return_value.content = b"compressed_data" + + # Act + self.tool.install_tool("Darwin", self.agent_temp_dir, "8.0.0") + + # Assert + mock_requests.assert_called_once_with( + "https://github.com/gitleaks/gitleaks/releases/download/v8.0.0/gitleaks_8.0.0_darwin_x64.tar.gz", + allow_redirects=True + ) + mock_file.assert_called_once_with(f"{self.agent_temp_dir}/gitleaks_8.0.0_darwin_x64.tar.gz", "wb") + mock_extract.assert_called_once() + + @patch("subprocess.run") + @patch("re.search") + def test_tool_already_installed(self, mock_search, mock_run): + # Arrange + mock_search.side_effect = [None, True, True] + mock_run.return_value = MagicMock(stdout="gitleaks version 8.20.0") + + # Act + self.tool.install_tool("Linux", self.agent_temp_dir, "8.20.0") + + # Assert + mock_run.assert_called_once_with(f"{self.agent_temp_dir}/gitleaks --version", capture_output=True, shell=True, text=True) + mock_search.assert_any_call(r"8.20.0", "gitleaks version 8.20.0") + + @patch("os.path.exists") + @patch("builtins.open", new_callable=mock_open, read_data='{"key": "value"}') + def test_extract_json_data(self, mock_file, mock_exists): + # Arrange + mock_exists.return_value = True + + # Act + result = self.tool._extract_json_data("/path/to/file.json") + + # Assert + self.assertEqual(result, {"key": "value"}) + mock_file.assert_called_once_with("/path/to/file.json", "r", encoding="utf-8") + + @patch("os.path.exists") + def test_extract_json_data_file_not_exists(self, mock_exists): + # Arrange + mock_exists.return_value = False + + # Act + result = self.tool._extract_json_data("/path/to/nonexistent.json") + + # Assert + self.assertEqual(result, []) + + @patch("builtins.open", new_callable=mock_open) + def test_create_report(self, mock_file): + # Arrange + data = [{"key": "value"}] + + # Act + self.tool._create_report("/path/to/report.json", data) + + # Assert + mock_file.assert_called_once_with("/path/to/report.json", "w", encoding="utf-8") + + def test_check_path(self): + # Arrange + excluded_paths = ["excluded_dir"] + + # Act & Assert + self.assertTrue(self.tool._check_path("some/excluded_dir/file.txt", excluded_paths)) + self.assertFalse(self.tool._check_path("some/other_dir/file.txt", excluded_paths)) + + def test_add_flags(self): + config_tool = { + "gitleaks": { + "ALLOW_IGNORE_LEAKS": False, + "ENABLE_CUSTOM_RULES": True, + } + } + expected_flags = [ + "--ignore-gitleaks-allow", + "--config", + f"{self.agent_work_folder}{os.sep}rules{os.sep}gitleaks{os.sep}gitleaks.toml" + ] + result = self.tool._add_flags(config_tool, "gitleaks", self.agent_work_folder) + self.assertEqual(result, expected_flags) + + @patch("subprocess.run") + @patch("os.path.join", side_effect=lambda *args: "/".join(args)) + @patch("devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_tool.GitleaksTool._extract_json_data", return_value=[{"leak": "found"}]) + def test_run_tool_secret_scan_single_file(self, mock_extract, mock_join, mock_run): + # Act + findings, finding_path = self.tool.run_tool_secret_scan( + files=["file1.txt"], + agent_os="Linux", + agent_work_folder=self.agent_work_folder, + repository_name=self.repository_name, + config_tool=self.config_tool, + secret_tool=None, + secret_external_checks=None, + agent_temp_dir=self.agent_temp_dir, + tool="gitleaks" + ) + + # Assert + self.assertEqual(findings, [{"leak": "found"}]) + self.assertEqual(finding_path, f"{self.agent_work_folder}/gitleaks_report.json") + mock_run.assert_called_once() + + @patch("concurrent.futures.ThreadPoolExecutor") + @patch("subprocess.run") + @patch("os.path.join", side_effect=lambda *args: "/".join(args)) + @patch("devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_tool.GitleaksTool._extract_json_data", side_effect=lambda x: [{"leak": f"found in {x}"}]) + @patch("devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_tool.GitleaksTool._create_report") + def test_run_tool_secret_scan_multiple_files(self, mock_create_report, mock_extract, mock_join, mock_run, mock_executor): + # Arrange + files = ["file1.txt", "file2.txt"] + mock_executor.return_value.__enter__.return_value.submit.side_effect = lambda fn, *args, **kwargs: fn(*args, **kwargs) + + # Act + findings, finding_path = self.tool.run_tool_secret_scan( + files=files, + agent_os="Linux", + agent_work_folder=self.agent_work_folder, + repository_name=self.repository_name, + config_tool=self.config_tool, + secret_tool=None, + secret_external_checks=None, + agent_temp_dir=self.agent_temp_dir, + tool="gitleaks" + ) + + # Assert + self.assertEqual(len(findings), 2) + self.assertIn({"leak": "found in /path/to/work/folder/gitleaks_aux_report_file1.txt.json"}, findings) + self.assertIn({"leak": "found in /path/to/work/folder/gitleaks_aux_report_file2.txt.json"}, findings) + self.assertEqual(finding_path, f"{self.agent_work_folder}/gitleaks_report.json") diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_deserealizator.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_deserealizator.py old mode 100644 new mode 100755 index 04e235b3c..00d2c393e --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_deserealizator.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_deserealizator.py @@ -22,7 +22,24 @@ def test_get_list_vulnerability(self): } } }, - "Raw": "secret" + "Raw": "secret", + "Id": "SECRET_SCANNING" + }, + { + "DetectorName": "ExampleDetector", + "SourceMetadata": { + "Data": { + "Filesystem": { + "line": 20, + "file": "/path/to/file.py" + } + } + }, + "ExtraData": { + "name" : "ActuatorRule" + }, + "Raw": "management.endpoints.web.exposure.include=env,heapdump,threaddump,loggers", + "Id": "MISCONFIGURATION_SCANNING" } ] @@ -30,19 +47,32 @@ def test_get_list_vulnerability(self): vulnerabilities = self.deserealizator.get_list_vulnerability(results_scan_list, "Linux", "/path/to", ) # Assertions - self.assertEqual(len(vulnerabilities), 1) - vulnerability = vulnerabilities[0] - self.assertIsInstance(vulnerability, Finding) - self.assertEqual(vulnerability.id, "SECRET_SCANNING") - self.assertIsNone(vulnerability.cvss) - self.assertEqual(vulnerability.where, "/file.py, Secret: sec*********ret") - self.assertEqual(vulnerability.description, "Sensitive information in source code") - self.assertEqual(vulnerability.severity, "critical") - self.assertEqual(vulnerability.identification_date, datetime.now().strftime("%d%m%Y")) - self.assertEqual(vulnerability.module, "engine_secret") - self.assertEqual(vulnerability.category, Category.VULNERABILITY) - self.assertEqual(vulnerability.requirements, "ExampleDetector") - self.assertEqual(vulnerability.tool, "Trufflehog") + self.assertEqual(len(vulnerabilities), 2) + vulnerabilitySecret = vulnerabilities[0] + self.assertIsInstance(vulnerabilitySecret, Finding) + self.assertEqual(vulnerabilitySecret.id, "SECRET_SCANNING") + self.assertIsNone(vulnerabilitySecret.cvss) + self.assertEqual(vulnerabilitySecret.where, "/file.py, Secret: sec*********ret") + self.assertEqual(vulnerabilitySecret.description, "Sensitive information in source code") + self.assertEqual(vulnerabilitySecret.severity, "critical") + self.assertEqual(vulnerabilitySecret.identification_date, datetime.now().strftime("%d%m%Y")) + self.assertEqual(vulnerabilitySecret.module, "engine_secret") + self.assertEqual(vulnerabilitySecret.category, Category.VULNERABILITY) + self.assertEqual(vulnerabilitySecret.requirements, "ExampleDetector") + self.assertEqual(vulnerabilitySecret.tool, "Trufflehog") + vulnerabilityActuator = vulnerabilities[1] + self.assertIsInstance(vulnerabilityActuator, Finding) + self.assertEqual(vulnerabilityActuator.id, "MISCONFIGURATION_SCANNING") + self.assertIsNone(vulnerabilityActuator.cvss) + self.assertEqual(vulnerabilityActuator.where, "/file.py, Misconfiguration: man*********ers") + self.assertEqual(vulnerabilityActuator.description, "Actuator misconfiguration can leak sensitive information") + self.assertEqual(vulnerabilityActuator.severity, "critical") + self.assertEqual(vulnerabilityActuator.identification_date, datetime.now().strftime("%d%m%Y")) + self.assertEqual(vulnerabilityActuator.module, "engine_secret") + self.assertEqual(vulnerabilityActuator.category, Category.VULNERABILITY) + self.assertEqual(vulnerabilityActuator.requirements, "ExampleDetector") + self.assertEqual(vulnerabilityActuator.tool, "Trufflehog") + def test_get_where_correctly_linux(self): with patch.dict('os.environ', {'AGENT_OS': 'Linux'}): diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py old mode 100644 new mode 100755 index 2c6aaebd1..91172634a --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py @@ -1,50 +1,81 @@ -import json import unittest from unittest.mock import patch, MagicMock -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import DeserializeConfigTool from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.trufflehog.trufflehog_run import TrufflehogRun import os class TestTrufflehogRun(unittest.TestCase): + @patch('devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.trufflehog.trufflehog_run.subprocess.run') + def test_install_tool_win(self, mock_subprocess_run): + tool_version = "3.0.1" + agent_os = "Windows" + agent_temp_dir = "/tmp" + + mock_subprocess_run.return_value = MagicMock(stderr=b"version 3.0.1") + + obj = TrufflehogRun() + obj.run_install_win = MagicMock() + obj.run_install = MagicMock() + + obj.install_tool(agent_os, agent_temp_dir, tool_version) + + obj.run_install.assert_not_called() + obj.run_install_win.assert_not_called() @patch('devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.trufflehog.trufflehog_run.subprocess.run') def test_install_tool_unix(self, mock_subprocess_run): - os_patch = patch.dict('os.environ', {'AGENT_OS': 'Linux'}) - os_patch.start() - self.addCleanup(os_patch.stop) + tool_version = "3.0.1" + agent_os = "Linux" + agent_temp_dir = "/tmp" - mock_subprocess_run.return_value.stdout = b'Trufflehog version 1.0.0' - mock_subprocess_run.return_value.stderr = b'' + mock_subprocess_run.return_value = MagicMock(stderr=b"version 2.9.0") - trufflehog_run = TrufflehogRun() - trufflehog_run.install_tool("Linux", "/tmp") + obj = TrufflehogRun() + obj.run_install_win = MagicMock() + obj.run_install = MagicMock() - mock_subprocess_run.assert_called_once_with("trufflehog --version", capture_output=True, shell=True) + obj.install_tool(agent_os, agent_temp_dir, tool_version) + + obj.run_install.assert_called_once_with(tool_version) + obj.run_install_win.assert_not_called() + @patch('devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.trufflehog.trufflehog_run.subprocess.run') + def test_install_tool_unix_no_install(self, mock_subprocess_run): + tool_version = "3.0.1" + agent_os = "Linux" + agent_temp_dir = "/tmp" + + mock_subprocess_run.return_value = MagicMock(stderr=f"version {tool_version}".encode('utf-8')) + + obj = TrufflehogRun() + obj.run_install_win = MagicMock() + obj.run_install = MagicMock() + + obj.install_tool(agent_os, agent_temp_dir, tool_version) + + obj.run_install.assert_not_called() + obj.run_install_win.assert_not_called() + @patch('subprocess.run') def test_run_install(self, mock_subprocess_run): + tool_version = "1.2.3" trufflehog_run = TrufflehogRun() - trufflehog_run.run_install() + trufflehog_run.run_install(tool_version) mock_subprocess_run.assert_called_once_with( - "curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin", + f"curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin v{tool_version}", capture_output=True, shell=True ) @patch('devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.trufflehog.trufflehog_run.subprocess.Popen') def test_run_install_win(self, mock_popen): - + agent_temp_dir = "C:/temp" + tool_version = "1.2.3" trufflehog_run = TrufflehogRun() - trufflehog_run.run_install_win("C:/temp") + trufflehog_run.run_install_win(agent_temp_dir, tool_version) expected_command = ( - "powershell -Command " - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; [Net.ServicePointManager]::SecurityProtocol; " + - "New-Item -Path C:/temp -ItemType Directory -Force; " + - "Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh' -OutFile C:/temp\\install_trufflehog.sh; " + - "bash C:/temp\\install_trufflehog.sh -b C:/Trufflehog/bin; " + - "$env:Path += ';C:/Trufflehog/bin'; C:/Trufflehog/bin/trufflehog.exe --version" + f"powershell -Command [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; [Net.ServicePointManager]::SecurityProtocol; New-Item -Path {agent_temp_dir} -ItemType Directory -Force; Invoke-WebRequest -Uri 'https://github.com/trufflesecurity/trufflehog/releases/download/v{tool_version}/trufflehog_{tool_version}_windows_amd64.tar.gz' -OutFile {agent_temp_dir}/trufflehog.tar.gz -UseBasicParsing; tar -xzf {agent_temp_dir}/trufflehog.tar.gz -C {agent_temp_dir}; Remove-Item {agent_temp_dir}/trufflehog.tar.gz; $env:Path += '; + {agent_temp_dir}'; & {agent_temp_dir}/trufflehog.exe --version" ) mock_popen.assert_called_once_with(expected_command, stdout=-1, stderr=-1, shell=True) @@ -53,12 +84,13 @@ def test_run_install_win(self, mock_popen): @patch.object(TrufflehogRun, 'config_include_path') def test_run_tool_secret_scan(self, mock_config_include_path, mock_thread_pool_executor, mock_open): mock_executor = MagicMock() - mock_executor_map_result = ['{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}\n'] + mock_executor_map_result = ['{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null,"Id": "SECRET_SCANNING"}\n'] mock_executor.map.return_value = mock_executor_map_result mock_thread_pool_executor.return_value.__enter__.return_value = mock_executor mock_config_include_path.return_value = ['/usr/temp/includePath0.txt'] + agent_temp_dir = '/tmp' files_commits = ['/usr/file1.py', '/usr/file2.py'] agent_os = 'Windows' agent_work_folder = '/usr/temp' @@ -82,19 +114,27 @@ def test_run_tool_secret_scan(self, mock_config_include_path, mock_thread_pool_e }, "TARGET_BRANCHES": ["trunk", "develop", "main"], "trufflehog": { + "VERSION": "1.2.3", "EXCLUDE_PATH": [".git", "node_modules", "target", "build", "build.gradle", "twistcli-scan", ".svg", ".drawio"], "NUMBER_THREADS": 4, "ENABLE_CUSTOM_RULES" : "True", "EXTERNAL_DIR_OWNER": "External_Github", - "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks" + "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks", + "APP_ID_GITHUB":"123123", + "INSTALLATION_ID_GITHUB":"234234", + "RULES": { + "MISSCONFIGURATION_SCANNING" : { + "References" : "https://link.reference.com", + "Mitigation" : "Make sure do all good" + } + } } } - config_tool = DeserializeConfigTool(json_data=json_config_tool, tool="trufflehog") - secret_tool = "secret" + secret_tool = None trufflehog_run = TrufflehogRun() - result, file_findings = trufflehog_run.run_tool_secret_scan(files_commits, agent_os, agent_work_folder, repository_name, config_tool, secret_tool, secret_external_checks) + result, file_findings = trufflehog_run.run_tool_secret_scan(files_commits, agent_os, agent_work_folder, repository_name, json_config_tool, secret_tool, secret_external_checks, agent_temp_dir, "trufflehog") expected_result = [ {"SourceMetadata": {"Data": {"Filesystem": {"file": "/usr/bin/local/file1.txt", "line": 1}}}, "SourceID": 1, @@ -103,7 +143,10 @@ def test_run_tool_secret_scan(self, mock_config_include_path, mock_thread_pool_e "Raw": "https://admin:admin@the-internet.herokuapp.com", "RawV2": "https://admin:admin@the-internet.herokuapp.com/basic_auth", "Redacted": "https://admin:********@the-internet.herokuapp.com", "ExtraData": None, - "StructuredData": None}] + "StructuredData": None, + "Id": "SECRET_SCANNING", + 'References': 'N.A', + 'Mitigation': 'N.A'}] self.assertEqual(result, expected_result) self.assertEqual(os.path.normpath(file_findings), os.path.normpath(os.path.join('/usr/temp/', 'secret_scan_result.json'))) @@ -111,7 +154,7 @@ def test_run_tool_secret_scan(self, mock_config_include_path, mock_thread_pool_e def test_config_include_path(self, mock_open): trufflehog_run = TrufflehogRun() - result = trufflehog_run.config_include_path(['/usr/file1.py', '/usr/file2.py'], '/usr/temp') + result = trufflehog_run.config_include_path(['/usr/file1.py', '/usr/file2.py'], '/usr/temp', 'Windows') expected_result = ['/usr/temp/includePath0.txt', '/usr/temp/includePath1.txt'] self.assertEqual(result, expected_result) @@ -119,10 +162,10 @@ def test_config_include_path(self, mock_open): @patch('subprocess.run') def test_run_trufflehog_enable_rules_false(self, mock_subprocess_run): mock_subprocess_run.return_value.stdout.strip.return_value = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' - enable_custom_rules = "false" + enable_custom_rules = False trufflehog_run = TrufflehogRun() - result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules) + result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules, "trufflehog") expected_result = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' self.assertEqual(result, expected_result) @@ -130,10 +173,10 @@ def test_run_trufflehog_enable_rules_false(self, mock_subprocess_run): @patch('subprocess.run') def test_run_trufflehog_enable_rules_true(self, mock_subprocess_run): mock_subprocess_run.return_value.stdout.strip.return_value = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' - enable_custom_rules = "true" + enable_custom_rules = True trufflehog_run = TrufflehogRun() - result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules) + result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules, "trufflehog") expected_result = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' self.assertEqual(result, expected_result) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py old mode 100644 new mode 100755 index f29529416..dcc05e3f3 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py @@ -1,6 +1,5 @@ import unittest from unittest.mock import Mock, patch -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import DeserializeConfigTool from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.entry_points.entry_point_tool import engine_secret_scan class TestEngineSecretScan(unittest.TestCase): @@ -12,6 +11,7 @@ def test_engine_secret_scan(self, MockSetInputCore, MockSecretScan): mock_tool_gateway = Mock() mock_dict_args = { "remote_config_repo": "example_repo", + "remote_config_branch": "", "folder_path": ".", "environment": "test", "platform": "local", @@ -30,9 +30,7 @@ def test_engine_secret_scan(self, MockSetInputCore, MockSecretScan): } } json_config = { - "IGNORE_SEARCH_PATTERN": [ - "test" - ], + "IGNORE_SEARCH_PATTERN": "(.*test:*)", "MESSAGE_INFO_ENGINE_SECRET": "dummy message", "THRESHOLD": { "VULNERABILITY": { @@ -47,19 +45,28 @@ def test_engine_secret_scan(self, MockSetInputCore, MockSecretScan): }, "TARGET_BRANCHES": ["trunk", "develop", "main"], "trufflehog": { + "VERSION": "1.2.3", "EXCLUDE_PATH": [".git", "node_modules", "target", "build", "build.gradle", "twistcli-scan", ".svg", ".drawio"], "NUMBER_THREADS": 4, "ENABLE_CUSTOM_RULES" : "True", "EXTERNAL_DIR_OWNER": "External_Github", - "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks" + "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks", + "APP_ID_GITHUB":"123123", + "INSTALLATION_ID_GITHUB":"234234", + "RULES": { + "MISSCONFIGURATION_SCANNING" : { + "References" : "https://link.reference.com", + "Mitigation" : "Make sure do all good" + } + } } } - obj_config_tool = DeserializeConfigTool(json_config, 'trufflehog') mock_devops_platform_gateway.get_remote_config.side_effect = [json_exclusion ,json_config, json_exclusion] secret_tool = "secret" + skip_tool_isp = False mock_secret_scan_instance = MockSecretScan.return_value - mock_secret_scan_instance.complete_config_tool.return_value = obj_config_tool + mock_secret_scan_instance.complete_config_tool.return_value = json_config, skip_tool_isp mock_devops_platform_gateway.get_variable.side_effect = ["pipeline_name_carlos","pipeline_name_carlos", "pipeline_name", "build"] mock_secret_scan_instance.process.return_value = ([], "") diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/images_gateway.py b/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/images_gateway.py old mode 100644 new mode 100755 index 47203d1d1..7cb7e0328 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/images_gateway.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/images_gateway.py @@ -5,3 +5,7 @@ class ImagesGateway(metaclass=ABCMeta): @abstractmethod def list_images(self, image_to_scan) -> str: "get image to scan" + + @abstractmethod + def get_base_image(self, image_to_scan) -> str: + "get base image" diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/tool_gateway.py b/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/tool_gateway.py old mode 100644 new mode 100755 index 1361e87f2..6867d5485 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/tool_gateway.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/tool_gateway.py @@ -3,5 +3,5 @@ class ToolGateway(metaclass=ABCMeta): @abstractmethod - def run_tool_container_sca(self, dict_args, secret_tool, token_engine_container, scan_image, release) -> str: + def run_tool_container_sca(self, dict_args, secret_tool, token_engine_container, scan_image, release, base_image, exclusions, generate_sbom): "run tool container sca" diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/container_sca_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/container_sca_scan.py old mode 100644 new mode 100755 index d9c4cbb9e..76b392a4a --- a/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/container_sca_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/container_sca_scan.py @@ -18,19 +18,21 @@ def __init__( remote_config, tool_images: ImagesGateway, tool_deseralizator: DeseralizatorGateway, - build_id, + branch, secret_tool, token_engine_container, image_to_scan, + exclusions ): self.tool_run = tool_run self.remote_config = remote_config self.tool_images = tool_images self.tool_deseralizator = tool_deseralizator - self.build_id = build_id + self.branch = branch self.secret_tool = secret_tool self.token_engine_container = token_engine_container self.image_to_scan = image_to_scan + self.exclusions = exclusions def get_image(self, image_to_scan): """ @@ -41,6 +43,15 @@ def get_image(self, image_to_scan): """ return self.tool_images.list_images(image_to_scan) + def get_base_image(self, matching_image): + """ + Process the base image. + + Returns: + String: base image. + """ + return self.tool_images.get_base_image(matching_image) + def get_images_already_scanned(self): """ Create images scanned file if it does not exist and get the images that have already been scanned. @@ -66,21 +77,34 @@ def process(self): Returns: string: file scanning results name. """ - matching_image = self.get_image(self.image_to_scan) + base_image = None image_scanned = None + matching_image = self.get_image(self.image_to_scan) + if self.remote_config['GET_IMAGE_BASE']: + base_image = self.get_base_image(matching_image) + sbom_components = None + generate_sbom = self.remote_config["SBOM"]["ENABLED"] and any( + branch in str(self.branch) + for branch in self.remote_config["SBOM"]["BRANCH_FILTER"] + ) if matching_image: image_name = matching_image.tags[0] - result_file = image_name.replace("/","_") + "_scan_result.json" + result_file = image_name.replace("/", "_") + "_scan_result.json" if image_name in self.get_images_already_scanned(): print(f"The image {image_name} has already been scanned previously.") - return image_scanned - image_scanned = self.tool_run.run_tool_container_sca( - self.remote_config, self.secret_tool, self.token_engine_container, image_name, result_file + return image_scanned, base_image, sbom_components + image_scanned, sbom_components = self.tool_run.run_tool_container_sca( + self.remote_config, + self.secret_tool, + self.token_engine_container, + image_name, + result_file, base_image, self.exclusions, + generate_sbom, ) self.set_image_scanned(image_name) else: print(f"'Not image found for {self.image_to_scan}'. Tool skipped.") - return image_scanned + return image_scanned, base_image, sbom_components def deseralizator(self, image_scanned): """ diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/set_input_core.py b/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/set_input_core.py old mode 100644 new mode 100755 index 3d9be4f62..97c916901 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/set_input_core.py @@ -1,8 +1,7 @@ from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore -from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold - - from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions +from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold +from devsecops_engine_tools.engine_utilities.utils.utils import Utils class SetInputCore: @@ -13,25 +12,37 @@ def __init__(self, remote_config, exclusions, pipeline_name, tool, stage): self.tool = tool self.stage = stage - def get_exclusions(self, exclusions_data, pipeline_name, tool): - list_exclusions = [ - Exclusions( - id=item.get("id", ""), - where=item.get("where", ""), - cve_id=item.get("cve_id", ""), - create_date=item.get("create_date", ""), - expired_date=item.get("expired_date", ""), - severity=item.get("severity", ""), - hu=item.get("hu", ""), - reason=item.get("reason", "Risk acceptance"), - ) - for key, value in exclusions_data.items() - if key in {"All", pipeline_name} and value.get(tool) - for item in value[tool] - ] + def get_exclusions(self, exclusions_data, pipeline_name, tool, base_image): + list_exclusions = [] + print("The base image used is:", base_image) + for key, value in exclusions_data.items(): + if key not in {"All", pipeline_name} or not value.get(tool): + continue + + for item in value[tool]: + if key == "All": + source_images = item.get("source_images", []) + if source_images and base_image is None: + continue + if source_images and not any(base_image in source for source in source_images): + continue + + list_exclusions.append( + Exclusions( + id=item.get("id", ""), + where=item.get("where", ""), + cve_id=item.get("cve_id", ""), + create_date=item.get("create_date", ""), + expired_date=item.get("expired_date", ""), + severity=item.get("severity", ""), + hu=item.get("hu", ""), + reason=item.get("reason", "Risk acceptance"), + ) + ) + return list_exclusions - def set_input_core(self, image_scanned): + def set_input_core(self, image_scanned,base_image): """ Set the input core. @@ -43,8 +54,14 @@ def set_input_core(self, image_scanned): self.exclusions, self.pipeline_name, self.tool, + base_image + ), + Utils.update_threshold( + self, + Threshold(self.remote_config["THRESHOLD"]), + self.exclusions, + self.pipeline_name, ), - Threshold(self.remote_config["THRESHOLD"]), image_scanned, self.remote_config["MESSAGE_INFO_ENGINE_CONTAINER"], self.pipeline_name, diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/docker/docker_images.py b/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/docker/docker_images.py old mode 100644 new mode 100755 index 56b10f3b3..d5d91f1c1 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/docker/docker_images.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/docker/docker_images.py @@ -31,3 +31,20 @@ def list_images(self, image_to_scan): logger.error( f"Error listing images, docker must be running and added to PATH: {e}" ) + + def get_base_image(self, matching_image): + try: + client = docker.from_env() + image_details = client.api.inspect_image(matching_image.id) + labels = image_details.get("Config", {}).get("Labels", {}) + source_image = labels.get("x86.image.name") + if source_image: + logger.info(f"Base image for '{matching_image}' from source-image label: {source_image}") + return source_image + + logger.warning(f"Base image not found for '{matching_image}'.") + return None + + except Exception as e: + logger.error(f"Error getting base image: {e}") + return None \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/prisma_cloud/prisma_cloud_manager_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/prisma_cloud/prisma_cloud_manager_scan.py old mode 100644 new mode 100755 index 25d7c3161..280723964 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/prisma_cloud/prisma_cloud_manager_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/prisma_cloud/prisma_cloud_manager_scan.py @@ -4,9 +4,13 @@ import subprocess import logging import base64 +import json from devsecops_engine_tools.engine_sca.engine_container.src.domain.model.gateways.tool_gateway import ( ToolGateway, ) +from devsecops_engine_tools.engine_utilities.sbom.deserealizator import ( + get_list_component, +) from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings @@ -68,19 +72,81 @@ def scan_image( text=True, ) print(f"The image {image_name} was scanned") - return result_file except subprocess.CalledProcessError as e: logger.error(f"Error during image scan of {image_name}: {e.stderr}") + def _write_image_base(self, result_file, base_image, exclusions_data): + try: + with open(result_file, "r") as file: + data = json.load(file) + + prisma_exclusions = exclusions_data.get("All", {}).get("PRISMA", []) + modified = False + for result in data.get("results", []): + for vulnerability in result.get("vulnerabilities", []): + for exclusion in prisma_exclusions: + if ( + vulnerability.get("id") == exclusion.get("id") and + any(image.startswith(base_image) for image in exclusion.get("x86.image.name", [])) + ): + vulnerability["baseImage"] = base_image + modified = True + + if modified: + with open(result_file, "w") as file: + json.dump(data, file, indent=4) + except subprocess.CalledProcessError as e: + logger.error(f"Error during write image base of {base_image}: {e.stderr}") + + def _generate_sbom(self, image_scanned, remoteconfig, prisma_secret_key, image_name): + + url = f"{remoteconfig['PRISMA_CLOUD']['PRISMA_CONSOLE_URL']}/api/{remoteconfig['PRISMA_CLOUD']['PRISMA_API_VERSION']}/sbom/download/cli-images" + credentials = base64.b64encode( + f"{remoteconfig['PRISMA_CLOUD']['PRISMA_ACCESS_KEY']}:{prisma_secret_key}".encode() + ).decode() + headers = {"Authorization": f"Basic {credentials}"} + try: + + with open(image_scanned, "rb") as file: + image_object = file.read() + json_data = json.loads(image_object) + + if not json_data["results"]: + print("No results found in the scan, SBOM not generated") + return None + + response = requests.get( + url, + headers=headers, + params={ + "id": json_data["results"][0]["scanID"], + "sbomFormat": remoteconfig["PRISMA_CLOUD"]["SBOM_FORMAT"], + }, + ) + response.raise_for_status() + + result_sbom = f"{image_name.replace('/', '_')}_SBOM.json" + with open(result_sbom, "wb") as file: + file.write(response.content) + + print(f"SBOM generated and saved to: {result_sbom}") + + return get_list_component(result_sbom, remoteconfig["PRISMA_CLOUD"]["SBOM_FORMAT"]) + except Exception as e: + logger.error(f"Error generating SBOM: {e}") + def run_tool_container_sca( - self, remoteconfig, secret_tool, token_engine_container, image_name, result_file + self, remoteconfig, secret_tool, token_engine_container, image_name, result_file, base_image, exclusions, generate_sbom ): - prisma_secret_key = secret_tool["token_prisma_cloud"] if secret_tool else token_engine_container + prisma_secret_key = ( + secret_tool["token_prisma_cloud"] if secret_tool else token_engine_container + ) file_path = os.path.join( os.getcwd(), remoteconfig["PRISMA_CLOUD"]["TWISTCLI_PATH"] ) + sbom_components = None if not os.path.exists(file_path): self.download_twistcli( @@ -95,7 +161,16 @@ def run_tool_container_sca( image_name, result_file, remoteconfig, - prisma_secret_key, + prisma_secret_key ) + if base_image: + self._write_image_base(result_file, base_image, exclusions) + if generate_sbom: + sbom_components = self._generate_sbom( + image_scanned, + remoteconfig, + prisma_secret_key, + image_name + ) - return image_scanned + return image_scanned, sbom_components diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/trivy_tool/trivy_manager_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/trivy_tool/trivy_manager_scan.py old mode 100644 new mode 100755 index b869acaa4..1866e4292 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/trivy_tool/trivy_manager_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/driven_adapters/trivy_tool/trivy_manager_scan.py @@ -7,10 +7,15 @@ import requests import tarfile import zipfile +import json from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings +from devsecops_engine_tools.engine_utilities.sbom.deserealizator import ( + get_list_component, +) + logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() @@ -23,9 +28,9 @@ def download_tool(self, file, url): except Exception as e: logger.error(f"Error downloading trivy: {e}") - def install_tool(self, file, url): + def install_tool(self, file, url, command_prefix): installed = subprocess.run( - ["which", "./trivy"], + ["which", command_prefix], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) @@ -34,25 +39,30 @@ def install_tool(self, file, url): self.download_tool(file, url) with tarfile.open(file, 'r:gz') as tar_file: tar_file.extract(member=tar_file.getmember("trivy")) + return "./trivy" except Exception as e: logger.error(f"Error installing trivy: {e}") + else: + return installed.stdout.decode().strip() - def install_tool_windows(self, file, url): + def install_tool_windows(self, file, url, command_prefix): try: subprocess.run( - ["./trivy.exe", "--version"], + [command_prefix, "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) + return command_prefix except: try: self.download_tool(file, url) with zipfile.ZipFile(file, 'r') as zip_file: zip_file.extract(member="trivy.exe") + return "./trivy.exe" except Exception as e: logger.error(f"Error installing trivy: {e}") - def scan_image(self, prefix, image_name, result_file): + def scan_image(self, prefix, image_name, result_file, base_image): command = [ prefix, "--scanners", @@ -78,30 +88,57 @@ def scan_image(self, prefix, image_name, result_file): except Exception as e: logger.error(f"Error during image scan of {image_name}: {e}") - def run_tool_container_sca(self, remoteconfig, secret_tool, token_engine_container, image_name, result_file): + def _generate_sbom(self, prefix, image_name, remoteconfig): + result_sbom = f"{image_name.replace('/', '_')}_SBOM.json" + command = [ + prefix, + "image", + "--format", + remoteconfig["TRIVY"]["SBOM_FORMAT"], + "--output", + result_sbom + ] + command.extend(["--quiet", image_name]) + try: + subprocess.run( + command, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + print(f"SBOM generated and saved to: {result_sbom}") + + return get_list_component(result_sbom, remoteconfig["TRIVY"]["SBOM_FORMAT"]) + + except Exception as e: + logger.error(f"Error generating SBOM: {e}") + + def run_tool_container_sca(self, remoteconfig, secret_tool, token_engine_container, image_name, result_file, base_image, exclusions, generate_sbom): trivy_version = remoteconfig["TRIVY"]["TRIVY_VERSION"] os_platform = platform.system() arch_platform = platform.architecture()[0] base_url = f"https://github.com/aquasecurity/trivy/releases/download/v{trivy_version}/" + sbom_components = None + command_prefix = "trivy" if os_platform == "Linux": file=f"trivy_{trivy_version}_Linux-{arch_platform}.tar.gz" - self.install_tool(file, base_url+file) - command_prefix = "./trivy" + command_prefix = self.install_tool(file, base_url+file, "trivy") elif os_platform == "Darwin": file=f"trivy_{trivy_version}_macOS-{arch_platform}.tar.gz" - self.install_tool(file, base_url+file) - command_prefix = "./trivy" + command_prefix = self.install_tool(file, base_url+file, "trivy") elif os_platform == "Windows": file=f"trivy_{trivy_version}_windows-{arch_platform}.zip" - self.install_tool_windows(file, base_url+file) - command_prefix = "./trivy.exe" + command_prefix = self.install_tool_windows(file, base_url+file, "trivy.exe") else: logger.warning(f"{os_platform} is not supported.") return None image_scanned = ( - self.scan_image(command_prefix, image_name, result_file) + self.scan_image(command_prefix, image_name, result_file, base_image) ) + if generate_sbom: + sbom_components = self._generate_sbom(command_prefix, image_name, remoteconfig) - return image_scanned + return image_scanned, sbom_components diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/entry_points/entry_point_tool.py b/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/entry_points/entry_point_tool.py old mode 100644 new mode 100755 index 741177c68..dbe45d163 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/entry_points/entry_point_tool.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/src/infrastructure/entry_points/entry_point_tool.py @@ -23,10 +23,10 @@ def init_engine_sca_rm( tool, ): remote_config = tool_remote.get_remote_config( - dict_args["remote_config_repo"], "engine_sca/engine_container/ConfigTool.json" + dict_args["remote_config_repo"], "engine_sca/engine_container/ConfigTool.json", dict_args["remote_config_branch"] ) exclusions = tool_remote.get_remote_config( - dict_args["remote_config_repo"], "engine_sca/engine_container/Exclusions.json" + dict_args["remote_config_repo"], "engine_sca/engine_container/Exclusions.json", dict_args["remote_config_branch"] ) pipeline_name = tool_remote.get_variable("pipeline_name") handle_remote_config_patterns = HandleRemoteConfigPatterns( @@ -34,10 +34,12 @@ def init_engine_sca_rm( ) skip_flag = handle_remote_config_patterns.skip_from_exclusion() scan_flag = handle_remote_config_patterns.ignore_analysis_pattern() - build_id = tool_remote.get_variable("build_id") + branch = tool_remote.get_variable("branch_tag") stage = tool_remote.get_variable("stage") image_to_scan = dict_args["image_to_scan"] image_scanned = None + base_image = None + sbom_components = None deseralized = [] input_core = SetInputCore(remote_config, exclusions, pipeline_name, tool, stage) if scan_flag and not (skip_flag): @@ -46,17 +48,18 @@ def init_engine_sca_rm( remote_config, tool_images, tool_deseralizator, - build_id, + branch, secret_tool, dict_args["token_engine_container"], image_to_scan, + exclusions ) - image_scanned = container_sca_scan.process() + image_scanned, base_image, sbom_components = container_sca_scan.process() if image_scanned: deseralized = container_sca_scan.deseralizator(image_scanned) else: print("Tool skipped by DevSecOps policy") - logger.info("Tool skipped by DevSecOps policy") - core_input = input_core.set_input_core(image_scanned) + dict_args["send_metrics"] = "false" + core_input = input_core.set_input_core(image_scanned,base_image) - return deseralized, core_input + return deseralized, core_input, sbom_components diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/test/applications/test_runner_container_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_container/test/applications/test_runner_container_scan.py index 6051922a3..4f880945f 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/test/applications/test_runner_container_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/test/applications/test_runner_container_scan.py @@ -9,7 +9,7 @@ def test_init_engine_container(): with patch( "devsecops_engine_tools.engine_sca.engine_container.src.applications.runner_container_scan.init_engine_sca_rm" ) as mock_init_engine_sca_rm: - dict_args = {"remote_config_repo": "remote_repo"} + dict_args = {"remote_config_repo": "remote_repo", "remote_config_branch": ""} token = "token" tool = "PRISMA" diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_container_sca_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_container_sca_scan.py old mode 100644 new mode 100755 index 609a42420..8f8b6cec0 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_container_sca_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_container_sca_scan.py @@ -3,6 +3,7 @@ from devsecops_engine_tools.engine_sca.engine_container.src.domain.usecases.container_sca_scan import ( ContainerScaScan, ) +from devsecops_engine_tools.engine_core.src.domain.model.component import Component @pytest.fixture @@ -40,7 +41,8 @@ def container_sca_scan( "1234", "token", "token_engine_container", - "image_to_scan" + "image_to_scan", + "exclusions", ) @@ -71,30 +73,62 @@ def test_set_image_scanned(container_sca_scan): def test_process_image_already_scanned(container_sca_scan): mock_image = MagicMock() mock_image.tags = ["my_image:1234"] - container_sca_scan.get_images_already_scanned = MagicMock() - container_sca_scan.get_image = MagicMock() - container_sca_scan.get_image.return_value = mock_image - container_sca_scan.get_images_already_scanned.return_value = [ - "my_image:1234" - ] - assert container_sca_scan.process() == None + container_sca_scan.get_image = MagicMock(return_value=mock_image) + container_sca_scan.get_base_image = MagicMock(return_value="base_image:latest") + container_sca_scan.get_images_already_scanned = MagicMock( + return_value=["my_image:1234"] + ) + container_sca_scan.tool_run = MagicMock() + container_sca_scan.set_image_scanned = MagicMock() + + image_scanned, base_image, components = container_sca_scan.process() + + assert image_scanned is None + assert base_image == "base_image:latest" + container_sca_scan.get_image.assert_called_once_with( + container_sca_scan.image_to_scan + ) + container_sca_scan.get_images_already_scanned.assert_called_once() + container_sca_scan.tool_run.run_tool_container_sca.assert_not_called() + container_sca_scan.set_image_scanned.assert_not_called() def test_process_image_not_already_scanned(container_sca_scan): mock_image = MagicMock() mock_image.tags = ["my_image:1234"] - container_sca_scan.get_images_already_scanned = MagicMock() - container_sca_scan.get_image = MagicMock() - container_sca_scan.get_image.return_value = mock_image - container_sca_scan.get_images_already_scanned.return_value = [ - "my_image_scan_result.json" - ] - container_sca_scan.tool_run.run_tool_container_sca.return_value = [ - "my_image:1234_scan_result.json" + + container_sca_scan.get_image = MagicMock(return_value=mock_image) + container_sca_scan.get_base_image = MagicMock(return_value="base_image:latest") + container_sca_scan.get_images_already_scanned = MagicMock(return_value=[]) + container_sca_scan.tool_run = MagicMock() + component_list = [ + Component("component1", "version1"), + Component("component2", "version2"), ] + container_sca_scan.tool_run.run_tool_container_sca.return_value = ( + "my_image:1234_scan_result.json", + component_list, + ) container_sca_scan.set_image_scanned = MagicMock() - assert container_sca_scan.process() == ["my_image:1234_scan_result.json"] + image_scanned, base_image, components = container_sca_scan.process() + + assert image_scanned == "my_image:1234_scan_result.json" + container_sca_scan.get_image.assert_called_once_with( + container_sca_scan.image_to_scan + ) + container_sca_scan.get_images_already_scanned.assert_called_once() + container_sca_scan.tool_run.run_tool_container_sca.assert_called_once_with( + container_sca_scan.remote_config, + container_sca_scan.secret_tool, + container_sca_scan.token_engine_container, + "my_image:1234", + "my_image:1234_scan_result.json", + "base_image:latest", + "exclusions", + False, + ) + container_sca_scan.set_image_scanned.assert_called_once_with("my_image:1234") def test_deserialize(container_sca_scan): diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_set_input_core.py b/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_set_input_core.py old mode 100644 new mode 100755 index 6d87255ff..81e302d32 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_set_input_core.py @@ -17,54 +17,52 @@ def mock_tool_remote(): def test_get_exclusions(mock_tool_remote): exclusions_data = { - "All": { - "PRISMA": [ - { - "id": "CVE-2023-5363", - "where": "all", - "create_date": "24012023", - "expired_date": "22092023", - "hu": "", - } - ] - }, - "repository_test": { - "PRISMA": [ - { - "id": "XRAY-N94", - "create_date": "24012023", - "expired_date": "31122023", - "hu": "", - } - ] - }, - "12345_ProyectoEjemplo_RM": { - "PRISMA": [ - { - "id": "CVE-2023-6237", - "cve_id": "CVE-2023-6237", - "expired_date": "21092022", - "create_date": "24012023", - "hu": "", - } - ] - }, - } - pipeline_name = "my_pipeline" + "All": { + "PRISMA": [ + { + "id": "CVE-2023-5363", + "where": "all", + "create_date": "24012023", + "expired_date": "22092023", + "hu": "", + "source_images": ["base_image:latest", "another_image:tag"], + } + ] + }, + "repository_test": { + "PRISMA": [ + { + "id": "XRAY-N94", + "create_date": "24012023", + "expired_date": "31122023", + "hu": "", + } + ] + }, + "12345_ProyectoEjemplo_RM": { + "PRISMA": [ + { + "id": "CVE-2023-6237", + "cve_id": "CVE-2023-6237", + "expired_date": "21092022", + "create_date": "24012023", + "hu": "", + } + ] + }, + } + pipeline_name = "repository_test" + base_image = "base_image:latest" - set_input_core = SetInputCore( - mock_tool_remote, None, pipeline_name, "PRISMA", "release" - ) + + set_input_core = SetInputCore(mock_tool_remote, None, pipeline_name, "PRISMA", "release") - exclusions = set_input_core.get_exclusions(exclusions_data, pipeline_name, "PRISMA") + exclusions = set_input_core.get_exclusions(exclusions_data, pipeline_name, "PRISMA", base_image) - assert len(exclusions) == 1 - assert isinstance(exclusions[0], Exclusions) - assert exclusions[0].id == "CVE-2023-5363" - assert exclusions[0].where == "all" - assert exclusions[0].create_date == "24012023" - assert exclusions[0].expired_date == "22092023" - assert exclusions[0].hu == "" + # Verificar resultados + assert len(exclusions) == 2 + assert exclusions[0].id == "CVE-2023-5363" + assert exclusions[1].id == "XRAY-N94" def test_get_exclusions_for_specific_pipeline(mock_tool_remote): @@ -82,11 +80,11 @@ def test_get_exclusions_for_specific_pipeline(mock_tool_remote): } } pipeline_name = "pipeline_specific" - + base_image = "base_image:latest" set_input_core = SetInputCore( mock_tool_remote, None, pipeline_name, "PRISMA", "release" ) - exclusions = set_input_core.get_exclusions(exclusions_data, pipeline_name, "PRISMA") + exclusions = set_input_core.get_exclusions(exclusions_data, pipeline_name, "PRISMA",base_image) assert len(exclusions) == 1 assert exclusions[0].id == "CVE-2024-1234" @@ -111,10 +109,10 @@ def test_get_exclusions_no_matching_exclusions(mock_tool_remote): } } pipeline_name = "my_pipeline" - + base_image = "base_image:latest" set_input_core = SetInputCore( mock_tool_remote, None, pipeline_name, "PRISMA", "release" ) - exclusions = set_input_core.get_exclusions(exclusions_data, pipeline_name, "PRISMA") + exclusions = set_input_core.get_exclusions(exclusions_data, pipeline_name, "PRISMA",base_image) assert len(exclusions) == 0 diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/docker/test_docker_images.py b/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/docker/test_docker_images.py old mode 100644 new mode 100755 index fd73e178a..b7c15c26b --- a/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/docker/test_docker_images.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/docker/test_docker_images.py @@ -10,7 +10,6 @@ def mock_docker_client(): with patch("docker.from_env") as mock: yield mock - def test_list_images(mock_docker_client): # Arrange docker_images = DockerImages() @@ -46,3 +45,123 @@ def test_list_images(mock_docker_client): assert result.attrs["Created"] == "2023-08-02T12:34:56.789Z" mock_docker_client.assert_called_once() mock_client.images.list.assert_called_once() + + +def test_list_images_no_matching_image(mock_docker_client): + + docker_images = DockerImages() + image_to_scan = "non_existent_image:latest" + + mock_client = MagicMock() + mock_docker_client.return_value = mock_client + + mock_image1 = MagicMock() + mock_image1.tags = ["some_image:latest"] + mock_image2 = MagicMock() + mock_image2.tags = ["another_image:latest"] + + mock_client.images.list.return_value = [mock_image1, mock_image2] + + + result = docker_images.list_images(image_to_scan) + + + assert result is None + mock_client.images.list.assert_called_once() + + +def test_list_images_exception(mock_docker_client): + + docker_images = DockerImages() + image_to_scan = "test_image:latest" + + mock_client = MagicMock() + mock_docker_client.side_effect = Exception("Docker not running") + + + result = docker_images.list_images(image_to_scan) + + + assert result is None + mock_docker_client.assert_called_once() + + +def test_get_base_image_parent_image(mock_docker_client): + + docker_images = DockerImages() + + mock_client = MagicMock() + mock_docker_client.return_value = mock_client + + matching_image = MagicMock() + matching_image.id = "image_id" + + parent_image_details = {"RepoTags": ["base_image:latest"]} + mock_client.api.inspect_image.side_effect = [ + {"Parent": "parent_id"}, + parent_image_details, + ] + + + result = docker_images.get_base_image(matching_image) + + + assert result == None + + +def test_get_base_image_source_label(mock_docker_client): + + docker_images = DockerImages() + + mock_client = MagicMock() + mock_docker_client.return_value = mock_client + + matching_image = MagicMock() + matching_image.id = "image_id" + + mock_client.api.inspect_image.return_value = { + "Config": {"Labels": {"x86.image.name": "source_image:1.0"}}, + } + + result = docker_images.get_base_image(matching_image) + + assert result == "source_image:1.0" + mock_client.api.inspect_image.assert_called_once_with("image_id") + + +def test_get_base_image_no_base_image(mock_docker_client): + + docker_images = DockerImages() + + mock_client = MagicMock() + mock_docker_client.return_value = mock_client + + matching_image = MagicMock() + matching_image.id = "image_id" + + mock_client.api.inspect_image.return_value = {"Config": {"Labels": {}}} + + + result = docker_images.get_base_image(matching_image) + + + assert result is None + mock_client.api.inspect_image.assert_called_once_with("image_id") + + +def test_get_base_image_exception(mock_docker_client): + + docker_images = DockerImages() + + mock_client = MagicMock() + mock_docker_client.return_value = mock_client + + matching_image = MagicMock() + matching_image.id = "image_id" + + mock_client.api.inspect_image.side_effect = Exception("Inspection failed") + + result = docker_images.get_base_image(matching_image) + + assert result is None + mock_client.api.inspect_image.assert_called_once_with("image_id") \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/prisma_cloud/test_prisma_cloud_manager_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/prisma_cloud/test_prisma_cloud_manager_scan.py old mode 100644 new mode 100755 index 741ab8a19..240165ee5 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/prisma_cloud/test_prisma_cloud_manager_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/prisma_cloud/test_prisma_cloud_manager_scan.py @@ -1,9 +1,12 @@ +import json +import subprocess from devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.prisma_cloud.prisma_cloud_manager_scan import ( PrismaCloudManagerScan, ) - -from unittest.mock import patch, Mock, MagicMock +from devsecops_engine_tools.engine_core.src.domain.model.component import Component +from unittest.mock import patch, Mock, MagicMock, mock_open, mock_open import pytest +import json @pytest.fixture @@ -108,23 +111,54 @@ def test_download_twistcli_failure(twistcli_instance, mock_requests_get): def test_scan_image_success(mock_remoteconfig): - with patch("builtins.print") as mock_print, patch( - "devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.prisma_cloud.prisma_cloud_manager_scan.subprocess.run" - ) as mock_run: + mock_file_data = '{"scanned_data": {"vulnerabilities": []}}' + + with patch("builtins.print") as mock_print, \ + patch("devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.prisma_cloud.prisma_cloud_manager_scan.subprocess.run") as mock_run, \ + patch("builtins.open", mock_open(read_data=mock_file_data)) as mock_file, \ + patch("json.dump") as mock_json_dump: + mock_run.return_value = MagicMock() mock_run.return_value.stdout = "" mock_run.return_value.stderr = "" + scan_manager = PrismaCloudManagerScan() + + result = scan_manager.scan_image( "file_path", "image_name", "result.json", mock_remoteconfig, - "prisma_secret_key", + "prisma_secret_key" ) + assert result == "result.json" + mock_run.assert_called_once_with( + ( + "file_path", + "images", + "scan", + "--address", + mock_remoteconfig["PRISMA_CLOUD"]["PRISMA_CONSOLE_URL"], + "--user", + mock_remoteconfig["PRISMA_CLOUD"]["PRISMA_ACCESS_KEY"], + "--password", + "prisma_secret_key", + "--output-file", + "result.json", + "--details", + "image_name" + ), + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + mock_print.assert_any_call("The image image_name was scanned") def test_run_tool_container_sca_success(mock_remoteconfig, mock_scan_image): @@ -138,7 +172,128 @@ def test_run_tool_container_sca_success(mock_remoteconfig, mock_scan_image): scan_manager = PrismaCloudManagerScan() result = scan_manager.run_tool_container_sca( - mock_remoteconfig, {"token_prisma_cloud": "token"}, "token_container", "image_name", "result.json" + mock_remoteconfig, + {"token_prisma_cloud": "token"}, + "token_container", + "image_name", + "result.json" , None , {"exclusions": "all"}, + True, ) + + assert result == ("result.json", None) - assert result == "result.json" + +def test_generate_sbom_success(): + with patch( + "builtins.open", + mock_open(read_data=json.dumps({"results": [{"scanID": "12345"}]})), + ), patch("requests.get") as mock_request: + + # Configurar los mocks + mock_response = MagicMock() + mock_response.content = b"fake sbom content" + mock_request.return_value = mock_response + + # Crear instancia de PrismaCloudManagerScan + prisma_scan = PrismaCloudManagerScan() + + # Datos de prueba + image_scanned = "image_scanned.json" + remoteconfig = { + "PRISMA_CLOUD": { + "PRISMA_CONSOLE_URL": "http://example.com", + "PRISMA_API_VERSION": "v1", + "PRISMA_ACCESS_KEY": "access_key", + "SBOM_FORMAT": "json", + } + } + prisma_secret_key = "secret_key" + image_name = "test_image" + + # Llamar a la función + result = prisma_scan._generate_sbom( + image_scanned, remoteconfig, prisma_secret_key, image_name + ) + + # Verificar que se llamaron las funciones esperadas + mock_request.assert_called_once_with( + "http://example.com/api/v1/sbom/download/cli-images", + headers={"Authorization": "Basic YWNjZXNzX2tleTpzZWNyZXRfa2V5"}, + params={"id": "12345", "sbomFormat": "json"}, + ) + assert result is not None + +def test_write_image_base_success(): + mock_file_data = json.dumps({ + "results": [ + { + "vulnerabilities": [ + {"id": "CVE-1234-5678", "other_field": "value"} + ] + } + ] + }) + exclusions_data = { + "All": { + "PRISMA": [ + { + "id": "CVE-1234-5678", + "x86.image.name": ["python:3.9"] + } + ] + } + } + with patch("builtins.open", mock_open(read_data=mock_file_data)) as mock_file, \ + patch("json.dump") as mock_json_dump: + scan_manager = PrismaCloudManagerScan() + scan_manager._write_image_base("result.json", "python:3.9", exclusions_data) + + # Validar que el archivo fue modificado + mock_file.assert_called_with("result.json", "w") + mock_json_dump.assert_called_once() + written_data = mock_json_dump.call_args[0][0] + assert written_data["results"][0]["vulnerabilities"][0]["baseImage"] == "python:3.9" + +def test_write_image_base_no_match(): + mock_file_data = json.dumps({ + "results": [ + { + "vulnerabilities": [ + {"id": "CVE-9999-8888", "other_field": "value"} + ] + } + ] + }) + exclusions_data = { + "All": { + "PRISMA": [ + { + "id": "CVE-1234-5678", + "source_images": ["python:3.9"] + } + ] + } + } + with patch("builtins.open", mock_open(read_data=mock_file_data)), \ + patch("json.dump") as mock_json_dump: + scan_manager = PrismaCloudManagerScan() + scan_manager._write_image_base("result.json", "python:3.9", exclusions_data) + + # Validar que el archivo no fue modificado + mock_json_dump.assert_not_called() + +def test_write_image_base_file_not_found(): + exclusions_data = { + "All": { + "PRISMA": [ + { + "id": "CVE-1234-5678", + "source_images": ["python:3.9"] + } + ] + } + } + with patch("builtins.open", side_effect=FileNotFoundError): + scan_manager = PrismaCloudManagerScan() + with pytest.raises(FileNotFoundError): + scan_manager._write_image_base("result.json", "python:3.9", exclusions_data) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/trivy_tool/test_trivy_manager_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/trivy_tool/test_trivy_manager_scan.py old mode 100644 new mode 100755 index 263eeda1f..1b066b09a --- a/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/trivy_tool/test_trivy_manager_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/driven_adapters/trivy_tool/test_trivy_manager_scan.py @@ -1,9 +1,11 @@ from devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.trivy_tool.trivy_manager_scan import ( TrivyScan, ) +from devsecops_engine_tools.engine_core.src.domain.model.component import Component from unittest.mock import patch, MagicMock, Mock import pytest +import subprocess @pytest.fixture @@ -40,7 +42,7 @@ def test_install_tool_success(trivy_scan_instance): mock_run.return_value = Mock(returncode=1) trivy_scan_instance.download_tool = MagicMock() - trivy_scan_instance.install_tool("file", "url") + trivy_scan_instance.install_tool("file", "url", "trivy") assert mock_tar_open.call_count == 1 @@ -52,7 +54,7 @@ def test_install_tool_exception(trivy_scan_instance): trivy_scan_instance.download_tool = MagicMock() trivy_scan_instance.download_tool.side_effect = Exception("custom error") - trivy_scan_instance.install_tool("file", "url") + trivy_scan_instance.install_tool("file", "url", "trivy") mocke_logger.assert_called_with("Error installing trivy: custom error") @@ -64,7 +66,7 @@ def test_install_tool_windows_success(trivy_scan_instance): mock_run.side_effect = Exception() trivy_scan_instance.download_tool = MagicMock() - trivy_scan_instance.install_tool_windows("file", "url") + trivy_scan_instance.install_tool_windows("file", "url", "trivy.exe") assert mock_zipfile.call_count == 1 @@ -77,7 +79,7 @@ def test_install_tool_windows_exception(trivy_scan_instance): trivy_scan_instance.download_tool = MagicMock() trivy_scan_instance.download_tool.side_effect = Exception("custom error") - trivy_scan_instance.install_tool_windows("file", "url") + trivy_scan_instance.install_tool_windows("file", "url", "trivy.exe") mocke_logger.assert_called_with("Error installing trivy: custom error") @@ -86,10 +88,10 @@ def test_scan_image_success(trivy_scan_instance): with patch("subprocess.run") as mock_run, patch( "builtins.print" ) as mock_print: - result = trivy_scan_instance.scan_image("prefix", "image_name", "result.json") + result = trivy_scan_instance.scan_image("prefix", "image_name", "result.json","base_image") assert mock_print.call_count == 1 - assert result == "result.json" + assert result == 'result.json' def test_scan_image_exception(trivy_scan_instance): @@ -98,7 +100,7 @@ def test_scan_image_exception(trivy_scan_instance): ) as mocke_logger: mock_run.side_effect = Exception("custom error") - trivy_scan_instance.scan_image("prefix", "image_name", "result.json") + trivy_scan_instance.scan_image("prefix", "image_name", "result.json","base_image") mocke_logger.assert_called_with("Error during image scan of image_name: custom error") @@ -114,10 +116,10 @@ def test_run_tool_container_sca_linux(trivy_scan_instance): file = f"trivy_{version}_Linux-64bit.tar.gz" base_url = f"https://github.com/aquasecurity/trivy/releases/download/v{version}/" - result = trivy_scan_instance.run_tool_container_sca(remote_config, None, None, "image_name", "result.json") + result = trivy_scan_instance.run_tool_container_sca(remote_config, None, None, "image_name", "result.json", "base_image", "exclusions", False) - trivy_scan_instance.install_tool.assert_called_with(file, base_url+file) - assert result == "result.json" + trivy_scan_instance.install_tool.assert_called_with(file, base_url+file, "trivy") + assert result == ("result.json", None) def test_run_tool_container_sca_darwin(trivy_scan_instance): @@ -127,14 +129,16 @@ def test_run_tool_container_sca_darwin(trivy_scan_instance): trivy_scan_instance.install_tool = MagicMock() trivy_scan_instance.scan_image = MagicMock() trivy_scan_instance.scan_image.return_value = "result.json" + trivy_scan_instance._generate_sbom = MagicMock() + trivy_scan_instance._generate_sbom.return_value = [Component("component1", "version1")] version = remote_config["TRIVY"]["TRIVY_VERSION"] file = f"trivy_{version}_macOS-64bit.tar.gz" base_url = f"https://github.com/aquasecurity/trivy/releases/download/v{version}/" - result = trivy_scan_instance.run_tool_container_sca(remote_config, None, None, "image_name", "result.json") + result = trivy_scan_instance.run_tool_container_sca(remote_config, None, None, "image_name", "result.json", "base_image","exclusions", True) - trivy_scan_instance.install_tool.assert_called_with(file, base_url+file) - assert result == "result.json" + trivy_scan_instance.install_tool.assert_called_with(file, base_url+file, "trivy") + assert result == ("result.json", [Component("component1", "version1")]) def test_run_tool_container_sca_windows(trivy_scan_instance): @@ -148,10 +152,10 @@ def test_run_tool_container_sca_windows(trivy_scan_instance): file = f"trivy_{version}_windows-64bit.zip" base_url = f"https://github.com/aquasecurity/trivy/releases/download/v{version}/" - result = trivy_scan_instance.run_tool_container_sca(remote_config, None, None, "image_name", "result.json") + result = trivy_scan_instance.run_tool_container_sca(remote_config, None, None, "image_name", "result.json", "base_image","exclusions", False) - trivy_scan_instance.install_tool_windows.assert_called_with(file, base_url+file) - assert result == "result.json" + trivy_scan_instance.install_tool_windows.assert_called_with(file, base_url+file, "trivy.exe") + assert result == ("result.json", None) def test_run_tool_container_sca_none(trivy_scan_instance): with patch("platform.system") as mock_platform, patch( @@ -160,7 +164,72 @@ def test_run_tool_container_sca_none(trivy_scan_instance): remote_config = {"TRIVY":{"TRIVY_VERSION": "1.2.3"}} mock_platform.return_value = "None" - result = trivy_scan_instance.run_tool_container_sca(remote_config, None, None, "image_name", "result.json") + result = trivy_scan_instance.run_tool_container_sca(remote_config, None, None, "image_name", "result.json", "base_image","exclusions", False) mock_logger.assert_called_with("None is not supported.") assert result == None + +@patch('devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.trivy_tool.trivy_manager_scan.subprocess.run') +@patch('devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.trivy_tool.trivy_manager_scan.get_list_component') +def test_generate_sbom_success(mock_get_list_component, mock_subprocess_run): + # Configurar los mocks + mock_subprocess_run.return_value = MagicMock() + mock_get_list_component.return_value = ["component1", "component2"] + + # Crear instancia de TrivyScan + trivy_scan = TrivyScan() + + # Datos de prueba + prefix = "trivy" + image_name = "test_image" + remoteconfig = { + "TRIVY": { + "SBOM_FORMAT": "json" + } + } + + # Llamar a la función + result = trivy_scan._generate_sbom(prefix, image_name, remoteconfig) + + # Verificar que se llamaron las funciones esperadas + mock_subprocess_run.assert_called_once_with( + [ + prefix, + "image", + "--format", + "json", + "--output", + f"{image_name.replace('/', '_')}_SBOM.json", + "--quiet", + image_name, + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + mock_get_list_component.assert_called_once_with(f"{image_name.replace('/', '_')}_SBOM.json", "json") + assert result, ["component1", "component2"] + +@patch('devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.trivy_tool.trivy_manager_scan.subprocess.run') +@patch('devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.trivy_tool.trivy_manager_scan.logger') +def test_generate_sbom_failure(mock_logger, mock_subprocess_run): + # Configurar los mocks + mock_subprocess_run.side_effect = Exception("Test exception") + + # Crear instancia de TrivyScan + trivy_scan = TrivyScan() + + # Datos de prueba + prefix = "trivy" + image_name = "test_image" + remoteconfig = { + "TRIVY": { + "SBOM_FORMAT": "json" + } + } + + # Llamar a la función y verificar que se lanza la excepción esperada + trivy_scan._generate_sbom(prefix, image_name, remoteconfig) + + mock_logger.error.assert_called_once_with("Error generating SBOM: Test exception") diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/entry_points/test_entry_point_tool.py b/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/entry_points/test_entry_point_tool.py old mode 100644 new mode 100755 index 6d9b3e8ce..b6eeca7c1 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/entry_points/test_entry_point_tool.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/test/infrastructure/entry_points/test_entry_point_tool.py @@ -2,7 +2,6 @@ init_engine_sca_rm, ) from unittest.mock import patch, Mock -import pytest def test_init_engine_sca_rm(): @@ -13,7 +12,7 @@ def test_init_engine_sca_rm(): ) as mock_set_input_core, patch( "devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.entry_points.entry_point_tool.HandleRemoteConfigPatterns" ) as mock_handle_remote_config_patterns: - dict_args = {"remote_config_repo": "remote_repo", "image_to_scan":"image"} + dict_args = {"remote_config_repo": "remote_repo", "image_to_scan":"image", "remote_config_branch": ""} token = "token" tool = "tool" mock_handle_remote_config_patterns.process_handle_working_directory.return_value = ( @@ -23,9 +22,9 @@ def test_init_engine_sca_rm(): mock_handle_remote_config_patterns.process_handle_analysis_pattern.return_value = ( True ) - mock_container_sca_scan.process.return_value = "scan_result.json" + mock_container_sca_scan.process.return_value = ("scan_result.json", None) - deserialized, core_input = init_engine_sca_rm( + deserialized, core_input, sbom_components = init_engine_sca_rm( Mock(), Mock(), Mock(), @@ -44,7 +43,7 @@ def test_init_engine_sca_rm_skip_tool(): ) as mock_set_input_core, patch( "devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.entry_points.entry_point_tool.HandleRemoteConfigPatterns" ) as mock_handle_remote_config_patterns: - dict_args = {"remote_config_repo": "remote_repo", "image_to_scan":"image"} + dict_args = {"remote_config_repo": "remote_repo", "image_to_scan":"image", "remote_config_branch": ""} token = "token" tool = "tool" mock_handle_remote_config_patterns.process_handle_working_directory.return_value = ( @@ -55,7 +54,7 @@ def test_init_engine_sca_rm_skip_tool(): True ) - deserialized, core_input = init_engine_sca_rm( + deserialized, core_input, sbom_components = init_engine_sca_rm( Mock(), Mock(), Mock(), @@ -76,7 +75,7 @@ def test_init_engine_sca_rm_no_exclusions(): ) as mock_set_input_core, patch( "devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.entry_points.entry_point_tool.HandleRemoteConfigPatterns" ) as mock_handle_remote_config_patterns: - dict_args = {"remote_config_repo": "remote_repo", "image_to_scan":"image"} + dict_args = {"remote_config_repo": "remote_repo", "image_to_scan":"image", "remote_config_branch": ""} token = "token" tool = "tool" mock_handle_remote_config_patterns.process_handle_working_directory.return_value = ( @@ -88,7 +87,7 @@ def test_init_engine_sca_rm_no_exclusions(): ) mock_container_sca_scan.process.return_value = "scan_result.json" - deserialized, core_input = init_engine_sca_rm( + deserialized, core_input, sbom_components = init_engine_sca_rm( Mock(), Mock(), Mock(), @@ -109,7 +108,7 @@ def test_init_engine_sca_rm_empty_remote_config(): ) as mock_set_input_core, patch( "devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.entry_points.entry_point_tool.HandleRemoteConfigPatterns" ) as mock_handle_remote_config_patterns: - dict_args = {"remote_config_repo": "remote_repo", "image_to_scan":"image"} + dict_args = {"remote_config_repo": "remote_repo", "image_to_scan":"image", "remote_config_branch": ""} token = "token" tool = "tool" mock_handle_remote_config_patterns.process_handle_working_directory.return_value = ( @@ -121,7 +120,7 @@ def test_init_engine_sca_rm_empty_remote_config(): ) mock_container_sca_scan.process.return_value = "scan_result.json" - deserialized, core_input = init_engine_sca_rm( + deserialized, core_input, sbom_components = init_engine_sca_rm( Mock(), Mock(), Mock(), diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/applications/runner_dependencies_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/applications/runner_dependencies_scan.py index 99ada9168..e5de62f4f 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/applications/runner_dependencies_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/applications/runner_dependencies_scan.py @@ -15,23 +15,23 @@ ) -def runner_engine_dependencies(dict_args, config_tool, secret_tool, devops_platform_gateway): +def runner_engine_dependencies( + dict_args, config_tool, secret_tool, devops_platform_gateway, sbom_tool_gateway +): try: tools_mapping = { - "XRAY": { - "tool_run": XrayScan, - "tool_deserializator": XrayDeserializator - }, + "XRAY": {"tool_run": XrayScan, "tool_deserializator": XrayDeserializator, "tool_sbom": sbom_tool_gateway}, "DEPENDENCY_CHECK": { "tool_run": DependencyCheckTool, - "tool_deserializator": DependencyCheckDeserialize - } + "tool_deserializator": DependencyCheckDeserialize, + "tool_sbom": sbom_tool_gateway + }, } selected_tool = config_tool["ENGINE_DEPENDENCIES"]["TOOL"] tool_run = tools_mapping[selected_tool]["tool_run"]() tool_deserializator = tools_mapping[selected_tool]["tool_deserializator"]() - + tool_sbom = tools_mapping[selected_tool]["tool_sbom"] return init_engine_dependencies( tool_run, @@ -39,7 +39,8 @@ def runner_engine_dependencies(dict_args, config_tool, secret_tool, devops_platf tool_deserializator, dict_args, secret_tool, - config_tool["ENGINE_DEPENDENCIES"]["TOOL"], + config_tool, + tool_sbom ) except Exception as e: diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/deserializator_gateway.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/deserializator_gateway.py index 3928f043d..18fb3894f 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/deserializator_gateway.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/deserializator_gateway.py @@ -4,5 +4,5 @@ class DeserializatorGateway(metaclass=ABCMeta): @abstractmethod - def get_list_findings(self, results_scan_file) -> "list[Finding]": + def get_list_findings(self, results_scan_file, remote_config) -> "list[Finding]": "Deserializator" diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py index 13ab5060d..4297d74ac 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py @@ -48,4 +48,4 @@ def deserializator(self, dependencies_scanned): Process the results deserializer. Terun: list: Deserialized list of findings. """ - return self.tool_deserializator.get_list_findings(dependencies_scanned) + return self.tool_deserializator.get_list_findings(dependencies_scanned, self.remote_config) diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/set_input_core.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/set_input_core.py index dc2dbe1be..c1b5145c5 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/set_input_core.py @@ -1,6 +1,7 @@ from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions +from devsecops_engine_tools.engine_utilities.utils.utils import Utils class SetInputCore: @@ -31,15 +32,6 @@ def get_exclusions(self, exclusions_data, pipeline_name, tool): list_exclusions.extend(exclusions) return list_exclusions - def update_threshold(self, threshold, exclusions_data, pipeline_name): - if (pipeline_name in exclusions_data) and ( - exclusions_data[pipeline_name].get("THRESHOLD", 0) - ): - threshold["VULNERABILITY"] = exclusions_data[pipeline_name][ - "THRESHOLD" - ].get("VULNERABILITY") - return threshold - def set_input_core(self, dependencies_scanned): """ Set the input core. @@ -53,10 +45,11 @@ def set_input_core(self, dependencies_scanned): self.pipeline_name, self.tool, ), - Threshold( - self.update_threshold( - self.remote_config["THRESHOLD"], self.exclusions, self.pipeline_name - ) + Utils.update_threshold( + self, + Threshold(self.remote_config["THRESHOLD"]), + self.exclusions, + self.pipeline_name, ), dependencies_scanned, self.remote_config["MESSAGE_INFO_ENGINE_DEPENDENCIES"], diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/dependency_check/dependency_check_deserialize.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/dependency_check/dependency_check_deserialize.py index 9dab014b5..ad46f2a89 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/dependency_check/dependency_check_deserialize.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/dependency_check/dependency_check_deserialize.py @@ -7,56 +7,149 @@ ) from dataclasses import dataclass from datetime import datetime -import json -import os +import xml.etree.ElementTree as ET +from packageurl import PackageURL +from cpe import CPE from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() + @dataclass class DependencyCheckDeserialize(DeserializatorGateway): + TOOL = "DEPENDENCY_CHECK" - def get_list_findings(self, dependencies_scanned_file) -> "list[Finding]": - filename, extension = os.path.splitext(dependencies_scanned_file) - if extension.lower() != ".json": - dependencies_scanned_file = f"{filename}.json" - - data_result = self.load_results(dependencies_scanned_file) + def get_list_findings(self, dependencies_scanned_file, remote_config) -> "list[Finding]": + dependencies, namespace = self.filter_vulnerabilities_by_confidence(dependencies_scanned_file, remote_config) list_open_vulnerabilities = [] - for dependency in data_result.get("dependencies", []): - for vulnerability in dependency.get("vulnerabilities", []): - vulnerable_software = vulnerability.get("vulnerableSoftware", []) - fix = ( - vulnerable_software[0] - .get("software", {}) - .get("versionEndExcluding", None) - if vulnerable_software + + for dependency in dependencies: + vulnerabilities_node = dependency.find('ns:vulnerabilities', namespace) + if vulnerabilities_node: + vulnerabilities = vulnerabilities_node.findall('ns:vulnerability', namespace) + for vulnerability in vulnerabilities: + fix = "Not found" + vulnerable_software = vulnerability.find('ns:vulnerableSoftware', namespace) + if vulnerable_software: + software = vulnerable_software.findall('ns:software', namespace) + if len(software) > 0: + fix = software[0].get("versionEndExcluding", "Not found").lower() + + id = vulnerability.find('ns:name', namespace).text[:28] + cvss = ", ".join(f"{child.tag.split('}')[-1]}: {child.text}" for child in vulnerability.find('ns:cvssV3', namespace)) if vulnerability.find('ns:cvssV3', namespace) else "" + where = self.get_where(dependency, namespace) + description = vulnerability.find('ns:description', namespace).text if vulnerability.find('ns:description', namespace).text else "" + severity = vulnerability.find('ns:severity', namespace).text.lower() + + finding_open = Finding( + id=id, + cvss=cvss, + where=where, + description=description[:120].replace("\n\n", " ").replace("\n", " ").strip() if len(description) > 0 else "No description available", + severity=severity, + identification_date=datetime.now().strftime("%d%m%Y"), + published_date_cve=None, + module="engine_dependencies", + category=Category.VULNERABILITY, + requirements=fix, + tool="DEPENDENCY_CHECK", + ) + list_open_vulnerabilities.append(finding_open) + + return list_open_vulnerabilities + + def filter_vulnerabilities_by_confidence(self, dependencies_scanned_file, remote_config): + data_result = ET.parse(dependencies_scanned_file) + root = data_result.getroot() + + namespace_uri = root.tag.split('}')[0].strip('{') + namespace = {'ns': namespace_uri} + ET.register_namespace('', namespace_uri) + + confidence_levels = ["low", "medium", "high", "highest"] + confidences = remote_config[self.TOOL]["VULNERABILITY_CONFIDENCE"] + + dependencies = root.find('ns:dependencies', namespace) + if dependencies: + to_remove = [] + for dep in dependencies.findall('ns:dependency', namespace): + identifiers = dep.find('ns:identifiers', namespace) + if identifiers: + vulnerability_ids = identifiers.findall('ns:vulnerabilityIds', namespace) + if vulnerability_ids: + vul_ids_confidences = [conf.get("confidence", "").lower() for conf in vulnerability_ids] + if len(vul_ids_confidences) > 0: + if not max(vul_ids_confidences, key=lambda c: confidence_levels.index(c)) in confidences: + to_remove.append(dep) + elif not "no_confidence" in confidences: + to_remove.append(dep) + for dep in to_remove: dependencies.remove(dep) + data_result.write(dependencies_scanned_file, encoding="utf-8", xml_declaration=True) + + return dependencies, namespace + + def get_where(self, dependency, namespace): + identifiers_node = dependency.find("ns:identifiers", namespace) + if identifiers_node: + package_node = identifiers_node.find(".//ns:package", namespace) + if package_node: + id = package_node.find("ns:id", namespace).text + purl = PackageURL.from_string(id) + purl_parts = purl.to_dict() + component_name = ( + purl_parts["namespace"] + ":" + if purl_parts["namespace"] + and len(purl_parts["namespace"]) > 0 + else "" + ) + component_name += ( + purl_parts["name"] + if purl_parts["name"] and len(purl_parts["name"]) > 0 + else "" + ) + component_name = component_name or None + component_version = ( + purl_parts["version"] + if purl_parts["version"] and len(purl_parts["version"]) > 0 + else "" + ) + return f"{component_name}:{component_version}" + + cpe_node = identifiers_node.find( + ".//ns:identifier[@type='cpe']", namespace + ) + if cpe_node: + id = cpe_node.find("ns:name", namespace).text + cpe = CPE(id) + component_name = ( + cpe.get_vendor()[0] + ":" + if len(cpe.get_vendor()) > 0 + else "" + ) + component_name += ( + cpe.get_product()[0] if len(cpe.get_product()) > 0 else "" + ) + component_name = component_name or None + component_version = ( + cpe.get_version()[0] + if len(cpe.get_version()) > 0 else None ) - finding_open = Finding( - id=vulnerability["name"][:20], - cvss=str(vulnerability.get("cvssv3", {})), - where=dependency.get("fileName").split(':')[-1].strip(), - description=vulnerability["description"][:170].replace("\n\n", " "), - severity=vulnerability["severity"].lower(), - identification_date=datetime.now().strftime("%d%m%Y"), - published_date_cve=None, - module="engine_dependencies", - category=Category.VULNERABILITY, - requirements=fix, - tool="DEPENDENCY_CHECK" + return f"{component_name}:{component_version}" + + maven_node = identifiers_node.find( + ".//ns:identifier[@type='maven']", namespace + ) + if maven_node: + maven_parts = maven_node.find("ns:name", namespace).text.split( + ":", ) - list_open_vulnerabilities.append(finding_open) - return list_open_vulnerabilities - - def load_results(self, dependencies_scanned_file): - try: - with open(dependencies_scanned_file) as f: - data = json.load(f) - return data - except Exception as ex: - logger.error(f"An error ocurred loading dependency-check results {ex}") - return None + if len(maven_parts) == 3: + component_name = maven_parts[0] + ":" + maven_parts[1] + component_version = maven_parts[2] + return f"{component_name}:{component_version}" + + return "" \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/dependency_check/dependency_check_tool.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/dependency_check/dependency_check_tool.py index 853bc2292..5a96a6574 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/dependency_check/dependency_check_tool.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/dependency_check/dependency_check_tool.py @@ -9,7 +9,9 @@ import shutil from devsecops_engine_tools.engine_utilities.utils.utils import Utils -from devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.helpers.get_artifacts import GetArtifacts +from devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.helpers.get_artifacts import ( + GetArtifacts, +) from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings @@ -17,12 +19,18 @@ class DependencyCheckTool(ToolGateway): + def __init__(self): + self.download_tool_called = False + def download_tool(self, cli_version): try: + self.download_tool_called = True url = f"https://github.com/jeremylong/DependencyCheck/releases/download/v{cli_version}/dependency-check-{cli_version}-release.zip" response = requests.get(url, allow_redirects=True) home_directory = os.path.expanduser("~") - zip_name = os.path.join(home_directory, f"dependency_check_{cli_version}.zip") + zip_name = os.path.join( + home_directory, f"dependency_check_{cli_version}.zip" + ) with open(zip_name, "wb") as f: f.write(response.content) @@ -39,7 +47,9 @@ def install_tool(self, cli_version, is_windows=False): return command_prefix home_directory = os.path.expanduser("~") - bin_route = os.path.join(home_directory, f"dependency-check/bin/{command_prefix}") + bin_route = os.path.join( + home_directory, f"dependency-check/bin/{command_prefix}" + ) if shutil.which(bin_route): return bin_route @@ -50,22 +60,33 @@ def install_tool(self, cli_version, is_windows=False): if os.path.exists(bin_route): if not is_windows: subprocess.run(["chmod", "+x", bin_route], check=True) - return bin_route + return bin_route except Exception as e: logger.error(f"Error installing OWASP dependency check: {e}") return None def scan_dependencies(self, command_prefix, file_to_scan, token): try: - command = [command_prefix, "--format", "JSON", "--format", "XML", "--nvdApiKey", token, "--scan", file_to_scan,] - - if not token: - print("¡¡Remember!!, it is recommended to use the API key for faster vulnerability database downloads.") - command = [command_prefix, "--format", "JSON", "--format", "XML", "--scan", file_to_scan,] - - subprocess.run(command, capture_output=True, check=True) - except subprocess.CalledProcessError as error: - logger.error(f"Error executing OWASP dependency check scan: {error}") + command = [ + command_prefix, + "--format", + "XML", + "--scan", + file_to_scan, + ] + + if token: + command.extend([ + "--nvdApiKey", + token + ]) + + if not self.download_tool: + command.append("--noupdate") + + result = subprocess.run(command, capture_output=True, check=True, text=True) + except subprocess.CalledProcessError as e: + logger.error(f"Error executing OWASP dependency check scan: {e.stderr}") def select_operative_system(self, cli_version): os_platform = platform.system() @@ -85,7 +106,7 @@ def search_result(self): except Exception as ex: logger.error(f"An error ocurred search dependency-check results {ex}") return None - + def is_java_installed(self): return shutil.which("java") is not None @@ -97,17 +118,21 @@ def run_tool_dependencies_sca( pipeline_name, to_scan, token, - token_engine_dependencies + token_engine_dependencies, ): if not self.is_java_installed(): - logger.error("Java is not installed, please install it to run dependency check") + logger.error( + "Java is not installed, please install it to run dependency check" + ) return None cli_version = remote_config["DEPENDENCY_CHECK"]["CLI_VERSION"] get_artifacts = GetArtifacts() - pattern = get_artifacts.excluded_files(remote_config, pipeline_name, exclusion, "DEPENDENCY_CHECK") + pattern = get_artifacts.excluded_files( + remote_config, pipeline_name, exclusion, "DEPENDENCY_CHECK" + ) to_scan = get_artifacts.find_artifacts( to_scan, pattern, remote_config["DEPENDENCY_CHECK"]["PACKAGES_TO_SCAN"] ) diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_deserialize_output.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_deserialize_output.py index f83ca89f2..3a1a792f8 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_deserialize_output.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_deserialize_output.py @@ -46,7 +46,7 @@ def set_list_finding(self, vul): ] return vulnerabilities - def get_list_findings(self, dependencies_scanned_file) -> "list[Finding]": + def get_list_findings(self, dependencies_scanned_file, remote_config) -> "list[Finding]": list_open_vulnerabilities = [] with open(dependencies_scanned_file, "rb") as file: json_data = json.loads(file.read()) diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_manager_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_manager_scan.py index 97fa54f28..e5493ed5d 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_manager_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_manager_scan.py @@ -9,7 +9,9 @@ import os import json -from devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.helpers.get_artifacts import GetArtifacts +from devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.helpers.get_artifacts import ( + GetArtifacts, +) from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings @@ -27,7 +29,11 @@ def install_tool_linux(self, prefix, version): if installed.returncode == 1: command = ["chmod", "+x", prefix] try: - url = f"https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/{version}/jfrog-cli-linux-amd64/jf" + architecture = platform.machine() + if architecture == "aarch64": + url = f"https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/{version}/jfrog-cli-linux-arm64/jf" + else: + url = f"https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/{version}/jfrog-cli-linux-amd64/jf" response = requests.get(url, allow_redirects=True) with open(prefix, "wb") as archivo: archivo.write(response.content) @@ -99,7 +105,7 @@ def config_audit_scan(self, to_scan): if os.path.exists(gradlew_path): os.chmod(gradlew_path, 0o755) - def scan_dependencies(self, prefix, cwd, mode, to_scan): + def scan_dependencies(self, prefix, cwd, config, mode, to_scan): command = [ prefix, mode, @@ -110,8 +116,7 @@ def scan_dependencies(self, prefix, cwd, mode, to_scan): command, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) if result.stdout or all( - word in result.stderr - for word in ["Technology", "WorkingDirectory", "Descriptors"] + word in result.stderr for word in config["XRAY"]["STDERR_EXPECTED_WORDS"] ): if result.stdout: scan_result = json.loads(result.stdout) @@ -119,7 +124,12 @@ def scan_dependencies(self, prefix, cwd, mode, to_scan): scan_result = {} if any( word in result.stderr - for word in ["What went wrong", "Caused by"] + for word in config["XRAY"]["STDERR_BREAK_ERRORS"] + ): + raise Exception(f"Error executing Xray scan: {result.stderr}") + if any( + word in result.stderr + for word in config["XRAY"]["STDERR_ACCEPTED_ERRORS"] ): logger.error(f"Error executing Xray scan: {result.stderr}") return None @@ -142,12 +152,14 @@ def run_tool_dependencies_sca( pipeline_name, to_scan, secret_tool, - token_engine_dependencies + token_engine_dependencies, ): token = secret_tool["token_xray"] if secret_tool else token_engine_dependencies if dict_args["xray_mode"] == "scan": get_artifacts = GetArtifacts() - pattern = get_artifacts.excluded_files(remote_config, pipeline_name, exclusion, "XRAY") + pattern = get_artifacts.excluded_files( + remote_config, pipeline_name, exclusion, "XRAY" + ) to_scan = get_artifacts.find_artifacts( to_scan, pattern, remote_config["XRAY"]["PACKAGES_TO_SCAN"] ) @@ -180,6 +192,7 @@ def run_tool_dependencies_sca( results_file = self.scan_dependencies( command_prefix, cwd, + remote_config, dict_args["xray_mode"], to_scan, ) diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/entry_points/entry_point_tool.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/entry_points/entry_point_tool.py index ee064f14e..1d9dd49ab 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/entry_points/entry_point_tool.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/entry_points/entry_point_tool.py @@ -7,9 +7,14 @@ from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.handle_remote_config_patterns import ( HandleRemoteConfigPatterns, ) +from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import ( + DevopsPlatformGateway, +) +from devsecops_engine_tools.engine_core.src.domain.model.gateway.sbom_manager import ( + SbomManagerGateway, +) import os -import sys from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings @@ -18,15 +23,23 @@ def init_engine_dependencies( - tool_run, tool_remote, tool_deserializator, dict_args, secret_tool, tool + tool_run, + tool_remote: DevopsPlatformGateway, + tool_deserializator, + dict_args, + secret_tool, + config_tool, + tool_sbom: SbomManagerGateway, ): remote_config = tool_remote.get_remote_config( dict_args["remote_config_repo"], "engine_sca/engine_dependencies/ConfigTool.json", + dict_args["remote_config_branch"] ) exclusions = tool_remote.get_remote_config( dict_args["remote_config_repo"], "engine_sca/engine_dependencies/Exclusions.json", + dict_args["remote_config_branch"] ) pipeline_name = tool_remote.get_variable("pipeline_name") @@ -38,7 +51,14 @@ def init_engine_dependencies( dependencies_scanned = None deserialized = [] - input_core = SetInputCore(remote_config, exclusions, pipeline_name, tool) + sbom_components = None + config_sbom = config_tool["SBOM_MANAGER"] + input_core = SetInputCore( + remote_config, + exclusions, + pipeline_name, + config_tool["ENGINE_DEPENDENCIES"]["TOOL"], + ) if scan_flag and not (skip_flag): to_scan = dict_args["folder_path"] if dict_args["folder_path"] else os.getcwd() @@ -53,6 +73,15 @@ def init_engine_dependencies( to_scan, secret_tool, ) + if config_sbom["ENABLED"] and any( + branch in str(tool_remote.get_variable("branch_tag")) + for branch in config_sbom["BRANCH_FILTER"] + ): + sbom_components = tool_sbom.get_components( + to_scan, + config_sbom, + pipeline_name + ) dependencies_scanned = dependencies_sca_scan.process() deserialized = ( dependencies_sca_scan.deserializator(dependencies_scanned) @@ -62,9 +91,9 @@ def init_engine_dependencies( else: logger.error(f"Path {to_scan} does not exist") else: - print(f"Tool skipped by DevSecOps policy") - logger.info(f"Tool skipped by DevSecOps policy") + print("Tool skipped by DevSecOps policy") + dict_args["send_metrics"] = "false" core_input = input_core.set_input_core(dependencies_scanned) - return deserialized, core_input + return deserialized, core_input, sbom_components diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/helpers/get_artifacts.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/helpers/get_artifacts.py index 49c336deb..4180a0b62 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/helpers/get_artifacts.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/helpers/get_artifacts.py @@ -9,11 +9,14 @@ logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() + class GetArtifacts: def excluded_files(self, remote_config, pipeline_name, exclusions, tool): pattern = remote_config[tool]["REGEX_EXPRESSION_EXTENSIONS"] - if pipeline_name in exclusions: + if (pipeline_name in exclusions) and ( + exclusions[pipeline_name].get(tool, None) + ): for ex in exclusions[pipeline_name][tool]: if ex.get("SKIP_FILES", 0): exclusion = ex.get("SKIP_FILES") @@ -29,7 +32,7 @@ def excluded_files(self, remote_config, pipeline_name, exclusions, tool): pattern = pattern2 return pattern - + def find_packages(self, pattern, packages, working_dir): packages_list = [] files_list = [] @@ -47,7 +50,7 @@ def find_packages(self, pattern, packages, working_dir): if extension_pattern.search(file): files_list.append(os.path.join(root, file)) return packages_list, files_list - + def compress_and_mv(self, tar_path, package): try: with tarfile.open(tar_path, "w") as tar: @@ -65,7 +68,7 @@ def move_files(self, dir_to_scan_path, finded_files): target = os.path.join(dir_to_scan_path, os.path.basename(file)) shutil.copy2(file, target) logger.debug(f"File to scan: {file}") - + def find_artifacts(self, to_scan, pattern, packages): dir_to_scan_path = os.path.join(to_scan, "dependencies_to_scan") if os.path.exists(dir_to_scan_path): diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/applications/test_runner_dependencies_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/applications/test_runner_dependencies_scan.py index 06cd1f32a..ce1b56ff7 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/applications/test_runner_dependencies_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/applications/test_runner_dependencies_scan.py @@ -9,13 +9,13 @@ def test_init_engine_dependencies_xray(): with patch( "devsecops_engine_tools.engine_sca.engine_dependencies.src.applications.runner_dependencies_scan.init_engine_dependencies" ) as mock_init_engine_dependencies: - dict_args = {"remote_config_repo": "remote_repo"} + dict_args = {"remote_config_repo": "remote_repo", "remote_config_branch": ""} token = "token" config_tool = { "ENGINE_DEPENDENCIES": {"ENABLED": "true", "TOOL": "XRAY"}, } - result = runner_engine_dependencies(dict_args, config_tool, token, None) + result = runner_engine_dependencies(dict_args, config_tool, token, None, None) mock_init_engine_dependencies.assert_any_call @@ -24,12 +24,12 @@ def test_init_engine_dependencies_dependency_check(): with patch( "devsecops_engine_tools.engine_sca.engine_dependencies.src.applications.runner_dependencies_scan.init_engine_dependencies" ) as mock_init_engine_dependencies: - dict_args = {"remote_config_repo": "remote_repo"} + dict_args = {"remote_config_repo": "remote_repo", "remote_config_branch": ""} token = "token" config_tool = { "ENGINE_DEPENDENCIES": {"ENABLED": "true", "TOOL": "DEPENDENCY_CHECK"}, } - result = runner_engine_dependencies(dict_args, config_tool, token, None) + result = runner_engine_dependencies(dict_args, config_tool, token, None, None) mock_init_engine_dependencies.assert_any_call diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/domain/usecases/test_dependencies_sca_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/domain/usecases/test_dependencies_sca_scan.py index b34e8e0f6..caf7589bb 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/domain/usecases/test_dependencies_sca_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/domain/usecases/test_dependencies_sca_scan.py @@ -66,7 +66,13 @@ def test_process(): dependencies_scan_instance.process() mock_tool_gateway.run_tool_dependencies_sca.assert_called_once_with( - remote_config, dict_args, exclusion, pipeline_name, to_scan, secret_tool, None + remote_config, + dict_args, + exclusion, + pipeline_name, + to_scan, + secret_tool, + None, ) @@ -97,5 +103,5 @@ def test_deserializator(): dependencies_scan_instance.deserializator(dependencies_scanned) mock_deserializator_gateway.get_list_findings.assert_called_once_with( - dependencies_scanned + dependencies_scanned, remote_config ) diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/dependency_check/test_dependency_check_deserialize.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/dependency_check/test_dependency_check_deserialize.py index 1200aff43..e6915a09f 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/dependency_check/test_dependency_check_deserialize.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/dependency_check/test_dependency_check_deserialize.py @@ -1,40 +1,162 @@ +import unittest +from unittest.mock import MagicMock, patch +import xml.etree.ElementTree as ET +from xml.etree.ElementTree import Element, SubElement from devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_deserialize import ( - DependencyCheckDeserialize, + DependencyCheckDeserialize ) -from unittest.mock import patch -import pytest - -@pytest.fixture -def deserializator(): - return DependencyCheckDeserialize() - - -@pytest.fixture -def json_data(): - return { - "dependencies": [ - { - "fileName": "path/to/package1:1.0", - "vulnerabilities": [ - { - "name": "CVE-1234", - "cvssv3": 7.5, - "description": "Una vulnerabilidad alta en package1.", - "severity": "HIGH" - } - ] + +class TestDependencyCheckDeserialize(unittest.TestCase): + @patch( + "xml.etree.ElementTree.parse" + ) + def test_filter_vulnerabilities_by_confidence(self, mock_parse): + # Arrange + xml_content = """ + + + + + + + + + + + + + + """ + + mock_parse.return_value = MagicMock() + mock_parse.return_value.getroot.return_value = ET.ElementTree(ET.fromstring(xml_content)).getroot() + remote_config = { + "DEPENDENCY_CHECK": { + "VULNERABILITY_CONFIDENCE": ["high"] + } + } + deserializer = DependencyCheckDeserialize() + + # Act + dependencies, namespace = deserializer.filter_vulnerabilities_by_confidence("test_file.xml", remote_config) + + # Assert + self.assertEqual(len(dependencies.findall('ns:dependency', namespace)), 1) + self.assertEqual(dependencies.findall('ns:dependency', namespace)[0].find('ns:identifiers/ns:vulnerabilityIds', namespace).attrib["confidence"], "high") + + + @patch( + "xml.etree.ElementTree.parse" + ) + def test_get_list_findings(self, mock_parse): + # Arrange + xml_content = """ + + + + + file_to_scan.tar: example.jar + + + + + + + CVE-2024-12345 + medium + Test vulnerability description + + + + + """ + + mock_parse.return_value = MagicMock() + mock_parse.return_value.getroot.return_value = ET.ElementTree(ET.fromstring(xml_content)).getroot() + remote_config = { + "DEPENDENCY_CHECK": { + "VULNERABILITY_CONFIDENCE": ["high", "medium"] } - ] - } - + } + deserializer = DependencyCheckDeserialize() + + # Act + result = deserializer.get_list_findings("test_file.xml", remote_config) + + # Assert + self.assertEqual(len(result), 1) + finding = result[0] + self.assertEqual(finding.id, "CVE-2024-12345") + self.assertEqual(finding.severity, "medium") + self.assertEqual(finding.description, "Test vulnerability description") + + def test_get_where_with_package(self): + # Arrange + xml_content = """ + + + + pkg:example_namespace/example_name@1.0.0 + + + """ + + # Act + dependency = ET.ElementTree(ET.fromstring(xml_content)).getroot() + result = DependencyCheckDeserialize().get_where(dependency, {"ns": "http://example.com/schema"}) + + # Assert + self.assertEqual(result, "example_name:1.0.0") + + @patch("cpe.CPE") + def test_get_where_with_cpe(self, MockCPE): + # Arrange + xml_content = """ + + + + cpe:/a:vendor:product:1.0.0 + + + """ + + MockCPE.return_value.get_vendor.return_value = ["vendor"] + MockCPE.return_value.get_product.return_value = ["product"] + MockCPE.return_value.get_version.return_value = ["1.0.0"] + + # Act + dependency = ET.ElementTree(ET.fromstring(xml_content)).getroot() + result = DependencyCheckDeserialize().get_where(dependency, {"ns": "http://example.com/schema"}) + + # Assert + self.assertEqual(result, "vendor:product:1.0.0") + + def test_get_where_with_maven(self): + # Arrange + xml_content = """ + + + + group:artifact:1.0.0 + + + """ + + # Act + dependency = ET.ElementTree(ET.fromstring(xml_content)).getroot() + result = DependencyCheckDeserialize().get_where(dependency, {"ns": "http://example.com/schema"}) + + # Assert + self.assertEqual(result, ("group:artifact:1.0.0")) -@patch.object(DependencyCheckDeserialize, 'load_results') -def test_get_list_findings_valid(mock_load_results, deserializator, json_data): - mock_load_results.return_value = json_data + def test_get_where_without_identifiers(self): + # Arrange + xml_content = """ + test""" - result = deserializator.get_list_findings("dummy_file.json") + # Act + dependency = ET.ElementTree(ET.fromstring(xml_content)).getroot() + result = DependencyCheckDeserialize().get_where(dependency, {"ns": "http://example.com/schema"}) - assert len(result) > 0 - assert result[0].id == "CVE-1234" - assert result[0].cvss == "7.5" - assert result[0].severity == "high" + # Assert + self.assertEqual(result, "") \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/dependency_check/test_dependency_check_tool.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/dependency_check/test_dependency_check_tool.py index d45eaceab..b3fcf8cdc 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/dependency_check/test_dependency_check_tool.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/dependency_check/test_dependency_check_tool.py @@ -3,17 +3,27 @@ import os import shutil import subprocess -from devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool import DependencyCheckTool +from devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool import ( + DependencyCheckTool, +) from devsecops_engine_tools.engine_utilities.utils.utils import Utils + class TestDependencyCheckTool(unittest.TestCase): - - @patch('requests.get') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.Utils') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.open', new_callable=mock_open) - @patch('os.path.join') - @patch('os.path.expanduser') - def test_download_tool(self, mock_expanduser, mock_path_join, mock_open, mock_utils, mock_requests_get): + + @patch("requests.get") + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.Utils" + ) + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.open", + new_callable=mock_open, + ) + @patch("os.path.join") + @patch("os.path.expanduser") + def test_download_tool( + self, mock_expanduser, mock_path_join, mock_open, mock_utils, mock_requests_get + ): mock_expanduser.return_value = "/mock/home" mock_path_join.return_value = "/mock/home/dependency_check_7.0.zip" mock_requests_get.return_value.content = b"Fake Zip Content" @@ -22,7 +32,10 @@ def test_download_tool(self, mock_expanduser, mock_path_join, mock_open, mock_ut tool.download_tool("7.0") - mock_requests_get.assert_called_with("https://github.com/jeremylong/DependencyCheck/releases/download/v7.0/dependency-check-7.0-release.zip", allow_redirects=True) + mock_requests_get.assert_called_with( + "https://github.com/jeremylong/DependencyCheck/releases/download/v7.0/dependency-check-7.0-release.zip", + allow_redirects=True, + ) mock_expanduser.assert_called_once() @@ -32,15 +45,27 @@ def test_download_tool(self, mock_expanduser, mock_path_join, mock_open, mock_ut mock_open().write.assert_called_once_with(b"Fake Zip Content") - mock_utils.return_value.unzip_file.assert_called_with("/mock/home/dependency_check_7.0.zip", "/mock/home") + mock_utils.return_value.unzip_file.assert_called_with( + "/mock/home/dependency_check_7.0.zip", "/mock/home" + ) - @patch('shutil.which') - @patch('os.path.exists') - @patch('os.path.join') - @patch('os.path.expanduser') - @patch('subprocess.run') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.download_tool') - def test_install_tool_already_installed(self, mock_download_tool, mock_subprocess_run, mock_expanduser, mock_path_join, mock_exists, mock_which): + @patch("shutil.which") + @patch("os.path.exists") + @patch("os.path.join") + @patch("os.path.expanduser") + @patch("subprocess.run") + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.download_tool" + ) + def test_install_tool_already_installed( + self, + mock_download_tool, + mock_subprocess_run, + mock_expanduser, + mock_path_join, + mock_exists, + mock_which, + ): mock_which.return_value = "/mock/path/dependency-check.sh" tool = DependencyCheckTool() @@ -51,16 +76,28 @@ def test_install_tool_already_installed(self, mock_download_tool, mock_subproces self.assertEqual(result, "dependency-check.sh") - @patch('shutil.which') - @patch('os.path.exists') - @patch('os.path.join') - @patch('os.path.expanduser') - @patch('subprocess.run') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.download_tool') - def test_install_tool_not_installed_linux(self, mock_download_tool, mock_subprocess_run, mock_expanduser, mock_path_join, mock_exists, mock_which): + @patch("shutil.which") + @patch("os.path.exists") + @patch("os.path.join") + @patch("os.path.expanduser") + @patch("subprocess.run") + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.download_tool" + ) + def test_install_tool_not_installed_linux( + self, + mock_download_tool, + mock_subprocess_run, + mock_expanduser, + mock_path_join, + mock_exists, + mock_which, + ): mock_which.side_effect = [None, None] mock_expanduser.return_value = "/mock/home" - mock_path_join.return_value = "/mock/home/dependency-check/bin/dependency-check.sh" + mock_path_join.return_value = ( + "/mock/home/dependency-check/bin/dependency-check.sh" + ) mock_exists.return_value = True tool = DependencyCheckTool() @@ -69,20 +106,35 @@ def test_install_tool_not_installed_linux(self, mock_download_tool, mock_subproc mock_download_tool.assert_called_once_with("7.0") - mock_subprocess_run.assert_called_once_with(["chmod", "+x", "/mock/home/dependency-check/bin/dependency-check.sh"], check=True) + mock_subprocess_run.assert_called_once_with( + ["chmod", "+x", "/mock/home/dependency-check/bin/dependency-check.sh"], + check=True, + ) self.assertEqual(result, "/mock/home/dependency-check/bin/dependency-check.sh") - @patch('shutil.which') - @patch('os.path.exists') - @patch('os.path.join') - @patch('os.path.expanduser') - @patch('subprocess.run') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.download_tool') - def test_install_tool_windows(self, mock_download_tool, mock_subprocess_run, mock_expanduser, mock_path_join, mock_exists, mock_which): + @patch("shutil.which") + @patch("os.path.exists") + @patch("os.path.join") + @patch("os.path.expanduser") + @patch("subprocess.run") + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.download_tool" + ) + def test_install_tool_windows( + self, + mock_download_tool, + mock_subprocess_run, + mock_expanduser, + mock_path_join, + mock_exists, + mock_which, + ): mock_which.side_effect = [None, None] mock_expanduser.return_value = "/mock/home" - mock_path_join.return_value = "/mock/home/dependency-check/bin/dependency-check.bat" + mock_path_join.return_value = ( + "/mock/home/dependency-check/bin/dependency-check.bat" + ) mock_exists.return_value = True tool = DependencyCheckTool() @@ -95,17 +147,32 @@ def test_install_tool_windows(self, mock_download_tool, mock_subprocess_run, moc self.assertEqual(result, "/mock/home/dependency-check/bin/dependency-check.bat") - @patch('shutil.which') - @patch('os.path.exists') - @patch('os.path.join') - @patch('os.path.expanduser') - @patch('subprocess.run') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.download_tool') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.logger.error') - def test_install_tool_error_handling(self, mock_logger_error, mock_download_tool, mock_subprocess_run, mock_expanduser, mock_path_join, mock_exists, mock_which): + @patch("shutil.which") + @patch("os.path.exists") + @patch("os.path.join") + @patch("os.path.expanduser") + @patch("subprocess.run") + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.download_tool" + ) + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.logger.error" + ) + def test_install_tool_error_handling( + self, + mock_logger_error, + mock_download_tool, + mock_subprocess_run, + mock_expanduser, + mock_path_join, + mock_exists, + mock_which, + ): mock_which.side_effect = [None, None] mock_expanduser.return_value = "/mock/home" - mock_path_join.return_value = "/mock/home/dependency-check/bin/dependency-check.sh" + mock_path_join.return_value = ( + "/mock/home/dependency-check/bin/dependency-check.sh" + ) mock_exists.return_value = True mock_subprocess_run.side_effect = Exception("chmod failed") @@ -115,11 +182,13 @@ def test_install_tool_error_handling(self, mock_logger_error, mock_download_tool mock_download_tool.assert_called_once_with("7.0") - mock_logger_error.assert_called_once_with("Error installing OWASP dependency check: chmod failed") + mock_logger_error.assert_called_once_with( + "Error installing OWASP dependency check: chmod failed" + ) self.assertIsNone(result) - @patch('subprocess.run') + @patch("subprocess.run") def test_scan_dependencies_success(self, mock_subprocess_run): mock_subprocess_run.return_value = MagicMock() @@ -128,16 +197,27 @@ def test_scan_dependencies_success(self, mock_subprocess_run): tool.scan_dependencies("dependency-check.sh", "mock_file_to_scan", "token") mock_subprocess_run.assert_called_once_with( - ['dependency-check.sh', '--format', 'JSON', '--format', 'XML', '--nvdApiKey', 'token', '--scan', 'mock_file_to_scan'], - capture_output=True, - check=True + [ + "dependency-check.sh", + "--format", + "XML", + "--scan", + "mock_file_to_scan", + "--nvdApiKey", + "token" + ], + capture_output=True, + check=True, + text=True ) - @patch('subprocess.run') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.logger.error') + @patch("subprocess.run") + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.logger.error" + ) def test_scan_dependencies_failure(self, mock_logger_error, mock_subprocess_run): mock_subprocess_run.side_effect = subprocess.CalledProcessError( - returncode=1, cmd="dependency-check.sh" + returncode=1, cmd="dependency-check.sh", stderr="Mock Error" ) tool = DependencyCheckTool() @@ -145,18 +225,31 @@ def test_scan_dependencies_failure(self, mock_logger_error, mock_subprocess_run) tool.scan_dependencies("dependency-check.sh", "mock_file_to_scan", "token") mock_logger_error.assert_called_once_with( - "Error executing OWASP dependency check scan: Command 'dependency-check.sh' returned non-zero exit status 1." + "Error executing OWASP dependency check scan: Mock Error" ) mock_subprocess_run.assert_called_once_with( - ['dependency-check.sh', '--format', 'JSON', '--format', 'XML', '--nvdApiKey', 'token', '--scan', 'mock_file_to_scan'], - capture_output=True, - check=True + [ + "dependency-check.sh", + "--format", + "XML", + "--scan", + "mock_file_to_scan", + "--nvdApiKey", + "token" + ], + capture_output=True, + check=True, + text=True ) - @patch('platform.system') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.install_tool') - def test_select_operative_system_linux(self, mock_install_tool, mock_platform_system): + @patch("platform.system") + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.install_tool" + ) + def test_select_operative_system_linux( + self, mock_install_tool, mock_platform_system + ): mock_platform_system.return_value = "Linux" tool = DependencyCheckTool() @@ -167,33 +260,45 @@ def test_select_operative_system_linux(self, mock_install_tool, mock_platform_sy self.assertEqual(result, mock_install_tool.return_value) - @patch('platform.system') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.install_tool') - def test_select_operative_system_windows(self, mock_install_tool, mock_platform_system): + @patch("platform.system") + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.install_tool" + ) + def test_select_operative_system_windows( + self, mock_install_tool, mock_platform_system + ): mock_platform_system.return_value = "Windows" - + tool = DependencyCheckTool() tool.select_operative_system("7.0") mock_install_tool.assert_called_once_with("7.0", is_windows=True) - @patch('platform.system') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.install_tool') - def test_select_operative_system_darwin(self, mock_install_tool, mock_platform_system): + @patch("platform.system") + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.install_tool" + ) + def test_select_operative_system_darwin( + self, mock_install_tool, mock_platform_system + ): mock_platform_system.return_value = "Darwin" - + tool = DependencyCheckTool() tool.select_operative_system("7.0") mock_install_tool.assert_called_once_with("7.0", is_windows=False) - @patch('platform.system') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.logger.warning') - def test_select_operative_system_unsupported(self, mock_logger_warning, mock_platform_system): + @patch("platform.system") + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.logger.warning" + ) + def test_select_operative_system_unsupported( + self, mock_logger_warning, mock_platform_system + ): mock_platform_system.return_value = "UnsupportedOS" - + tool = DependencyCheckTool() result = tool.select_operative_system("7.0") @@ -201,8 +306,8 @@ def test_select_operative_system_unsupported(self, mock_logger_warning, mock_pla mock_logger_warning.assert_called_once_with("UnsupportedOS is not supported.") self.assertIsNone(result) - - @patch('shutil.which') + + @patch("shutil.which") def test_is_java_installed_found(self, mock_which): mock_which.return_value = "/usr/bin/java" @@ -214,7 +319,7 @@ def test_is_java_installed_found(self, mock_which): self.assertTrue(result) - @patch('shutil.which') + @patch("shutil.which") def test_is_java_installed_not_found(self, mock_which): mock_which.return_value = None @@ -226,44 +331,89 @@ def test_is_java_installed_not_found(self, mock_which): self.assertFalse(result) - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.is_java_installed') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.logger.error') - def test_run_tool_dependencies_sca_java_not_installed(self, mock_logger_error, mock_is_java_installed): + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.is_java_installed" + ) + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.logger.error" + ) + def test_run_tool_dependencies_sca_java_not_installed( + self, mock_logger_error, mock_is_java_installed + ): mock_is_java_installed.return_value = False tool = DependencyCheckTool() - result = tool.run_tool_dependencies_sca({}, {}, {}, 'pipeline', 'to_scan', 'token', 'token_engine_dependencies') + result = tool.run_tool_dependencies_sca( + {}, {}, {}, "pipeline", "to_scan", "token", "token_engine_dependencies" + ) - mock_logger_error.assert_called_once_with("Java is not installed, please install it to run dependency check") + mock_logger_error.assert_called_once_with( + "Java is not installed, please install it to run dependency check" + ) self.assertIsNone(result) - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.is_java_installed') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.GetArtifacts') - def test_run_tool_dependencies_sca_no_artifacts_found(self, mock_get_artifacts, mock_is_java_installed): + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.is_java_installed" + ) + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.GetArtifacts" + ) + def test_run_tool_dependencies_sca_no_artifacts_found( + self, mock_get_artifacts, mock_is_java_installed + ): mock_is_java_installed.return_value = True mock_get_artifacts.return_value.find_artifacts.return_value = [] tool = DependencyCheckTool() - remote_config = {"DEPENDENCY_CHECK": {"CLI_VERSION": "7.0", "PACKAGES_TO_SCAN": "packages"}} - - result = tool.run_tool_dependencies_sca(remote_config, {}, {}, 'pipeline', 'to_scan', 'token', 'token_engine_dependencies') + remote_config = { + "DEPENDENCY_CHECK": {"CLI_VERSION": "7.0", "PACKAGES_TO_SCAN": "packages"} + } + + result = tool.run_tool_dependencies_sca( + remote_config, + {}, + {}, + "pipeline", + "to_scan", + "token", + "token_engine_dependencies", + ) self.assertIsNone(result) - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.is_java_installed') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.select_operative_system') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.scan_dependencies') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.search_result') - @patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.GetArtifacts') - def test_run_tool_dependencies_sca_success(self, mock_get_artifacts, mock_search_result, mock_scan_dependencies, mock_select_operative_system, mock_is_java_installed): + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.is_java_installed" + ) + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.select_operative_system" + ) + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.scan_dependencies" + ) + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.DependencyCheckTool.search_result" + ) + @patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.dependency_check.dependency_check_tool.GetArtifacts" + ) + def test_run_tool_dependencies_sca_success( + self, + mock_get_artifacts, + mock_search_result, + mock_scan_dependencies, + mock_select_operative_system, + mock_is_java_installed, + ): mock_is_java_installed.return_value = True mock_get_artifacts.return_value.excluded_files.return_value = "some_pattern" - mock_get_artifacts.return_value.find_artifacts.return_value = ["artifact_to_scan"] + mock_get_artifacts.return_value.find_artifacts.return_value = [ + "artifact_to_scan" + ] mock_select_operative_system.return_value = "dependency-check.sh" @@ -271,12 +421,24 @@ def test_run_tool_dependencies_sca_success(self, mock_get_artifacts, mock_search tool = DependencyCheckTool() - remote_config = {"DEPENDENCY_CHECK": {"CLI_VERSION": "7.0", "PACKAGES_TO_SCAN": "packages"}} - - result = tool.run_tool_dependencies_sca(remote_config, {}, {}, 'pipeline', 'to_scan', 'token', 'token_engine_dependencies') + remote_config = { + "DEPENDENCY_CHECK": {"CLI_VERSION": "7.0", "PACKAGES_TO_SCAN": "packages"} + } + + result = tool.run_tool_dependencies_sca( + remote_config, + {}, + {}, + "pipeline", + "to_scan", + "token", + "token_engine_dependencies", + ) mock_select_operative_system.assert_called_once_with("7.0") - mock_scan_dependencies.assert_called_once_with("dependency-check.sh", ["artifact_to_scan"], 'token_engine_dependencies') + mock_scan_dependencies.assert_called_once_with( + "dependency-check.sh", ["artifact_to_scan"], "token_engine_dependencies" + ) self.assertEqual(result, {"key": "value"}) diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/xray_tool/test_xray_deserialize_output.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/xray_tool/test_xray_deserialize_output.py index 6c1c10511..7047547c8 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/xray_tool/test_xray_deserialize_output.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/xray_tool/test_xray_deserialize_output.py @@ -42,5 +42,5 @@ def json_data(): def test_get_list_findings_valid(deserializator, json_data): with patch("builtins.open", mock_open(read_data=json.dumps(json_data))): - result = deserializator.get_list_findings("ruta_inexistente.json") + result = deserializator.get_list_findings("ruta_inexistente.json", {}) assert len(result) > 0 diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/xray_tool/test_xray_manager_scan.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/xray_tool/test_xray_manager_scan.py index 20c32ca02..a2fb174e5 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/xray_tool/test_xray_manager_scan.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/xray_tool/test_xray_manager_scan.py @@ -196,15 +196,24 @@ def test_scan_dependencies_success(xray_scan_instance): "os.path.join" ) as mock_path_join, patch( "os.getcwd" - ) as mock_os_getcwd: + ) as mock_os_getcwd, patch( + "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.driven_adapters.xray_tool.xray_manager_scan.logger.error" + ) as mock_logger: prefix = "jf" cwd = "working_dir" mode = "scan" to_scan = "target_file.tar" - mock_subprocess_run.return_value = Mock(returncode=0) + remote_config = { + "XRAY": { + "STDERR_EXPECTED_WORDS": ["expected"], + "STDERR_BREAK_ERRORS": ["break"], + "STDERR_ACCEPTED_ERRORS": ["accepted"] + } + } + mock_subprocess_run.return_value = Mock(returncode=0, stdout="", stderr="expected, accepted") mock_os_getcwd.return_value = "working_dir" - xray_scan_instance.scan_dependencies(prefix, cwd, mode, to_scan) + xray_scan_instance.scan_dependencies(prefix, cwd, remote_config, mode, to_scan) mock_subprocess_run.assert_called_with( [ @@ -219,6 +228,10 @@ def test_scan_dependencies_success(xray_scan_instance): text=True, ) + mock_logger.assert_called_with( + "Error executing Xray scan: expected, accepted" + ) + def test_scan_dependencies_failure(xray_scan_instance): with patch("subprocess.run") as mock_subprocess_run, patch( @@ -226,6 +239,7 @@ def test_scan_dependencies_failure(xray_scan_instance): ) as mock_logger_error: prefix = "jf" cwd = "working_dir" + remote_config = {"XRAY": {"STDERR_EXPECTED_WORDS": ["error"]}} mode = "scan" to_scan = "target_file.tar" mock_subprocess_run.return_value = Mock( @@ -234,7 +248,7 @@ def test_scan_dependencies_failure(xray_scan_instance): stdout="", ) - xray_scan_instance.scan_dependencies(prefix, cwd, mode, to_scan) + xray_scan_instance.scan_dependencies(prefix, cwd, remote_config, mode, to_scan) mock_logger_error.assert_called_with( "Error executing Xray scan: Command 'xray scan' returned non-zero exit status 1." @@ -275,13 +289,13 @@ def test_run_tool_dependencies_sca_linux(xray_scan_instance): pipeline_name, to_scan, secret_tool, - None + None, ) mock_install_tool.assert_called_with(prefix, "1.0") mock_config_server.assert_called_with(prefix, "token123") mock_scan_dependencies.assert_called_with( - prefix, "working_dir", dict_args["xray_mode"], "" + prefix, "working_dir", remote_config, dict_args["xray_mode"], "" ) @@ -305,7 +319,7 @@ def test_run_tool_dependencies_sca_windows(xray_scan_instance): dict_args = {"xray_mode": "audit"} prefix = os.path.join("user_path", "jf.exe") to_scan = "working_dir" - secret_tool = {"token_xray" : "token123"} + secret_tool = {"token_xray": "token123"} exclusion = {} pipeline_name = "pipeline" mock_system.return_value = "Windows" @@ -319,14 +333,14 @@ def test_run_tool_dependencies_sca_windows(xray_scan_instance): pipeline_name, to_scan, secret_tool, - None + None, ) mock_install_tool.assert_called_with(prefix, "1.0") mock_config_server.assert_called_with(prefix, "token123") mock_scan_dependencies.assert_called_with( - prefix, "working_dir", dict_args["xray_mode"], "" + prefix, "working_dir", remote_config, dict_args["xray_mode"], "" ) @@ -364,12 +378,12 @@ def test_run_tool_dependencies_sca_darwin(xray_scan_instance): pipeline_name, to_scan, token, - "token_container" + "token_container", ) mock_install_tool.assert_called_with(prefix, "1.0") mock_config_server.assert_called_with(prefix, "token_container") mock_scan_dependencies.assert_called_with( - prefix, "working_dir", dict_args["xray_mode"], "" + prefix, "working_dir", remote_config, dict_args["xray_mode"], "" ) diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/entry_points/test_entry_point_tool.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/entry_points/test_entry_point_tool.py index 90a1b9615..65f44aec0 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/entry_points/test_entry_point_tool.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/entry_points/test_entry_point_tool.py @@ -1,8 +1,9 @@ from devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.entry_points.entry_point_tool import ( init_engine_dependencies, ) - -from unittest.mock import patch, Mock +from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import DevopsPlatformGateway +from devsecops_engine_tools.engine_core.src.domain.model.gateway.sbom_manager import SbomManagerGateway +from unittest.mock import patch, Mock, MagicMock def test_init_engine_dependencies(): @@ -13,9 +14,9 @@ def test_init_engine_dependencies(): ) as mock_set_input_core, patch( "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.entry_points.entry_point_tool.HandleRemoteConfigPatterns" ) as mock_handle_remote_config_patterns: - dict_args = {"remote_config_repo": "remote_repo"} + dict_args = {"remote_config_repo": "remote_repo", "remote_config_branch": ""} token = "token" - tool = "tool" + tool = {"ENGINE_DEPENDENCIES": {"TOOL": "tool"}, "SBOM_MANAGER": {"ENABLED": True, "BRANCH_FILTER": ["trunk"]}} mock_handle_remote_config_patterns.process_handle_working_directory.return_value = ( "working_dir" ) @@ -32,4 +33,50 @@ def test_init_engine_dependencies(): dict_args, token, tool, + None ) + + +@patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.entry_points.entry_point_tool.HandleRemoteConfigPatterns') +@patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.entry_points.entry_point_tool.SetInputCore') +@patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.entry_points.entry_point_tool.DependenciesScan') +@patch('devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.entry_points.entry_point_tool.os.path.exists') +def test_init_engine_dependencies_success(mock_exists, mock_dependencies_scan, mock_set_input_core, mock_handle_remote_config_patterns): + # Configurar los mocks + mock_exists.return_value = True + mock_handle_remote_config_patterns.return_value.skip_from_exclusion.return_value = False + mock_handle_remote_config_patterns.return_value.ignore_analysis_pattern.return_value = True + mock_dependencies_scan.return_value.process.return_value = "scanned_dependencies" + mock_dependencies_scan.return_value.deserializator.return_value = ["deserialized_dependency"] + mock_set_input_core.return_value.set_input_core.return_value = "core_input" + + # Crear mocks para las dependencias + tool_run = MagicMock() + tool_remote = MagicMock(spec=DevopsPlatformGateway) + tool_remote.get_variable.return_value = "main" + tool_deserializator = MagicMock() + tool_sbom = MagicMock(spec=SbomManagerGateway) + dict_args = {"remote_config_repo": "repo", "folder_path": "path", "remote_config_branch": ""} + secret_tool = MagicMock() + config_tool = { + "SBOM_MANAGER": {"ENABLED": True, "BRANCH_FILTER": ["main"]}, + "ENGINE_DEPENDENCIES": {"TOOL": "tool"} + } + + # Llamar a la función + deserialized, core_input, sbom_components = init_engine_dependencies( + tool_run, tool_remote, tool_deserializator, dict_args, secret_tool, config_tool, tool_sbom + ) + + # Verificar que se llamaron las funciones esperadas + tool_remote.get_remote_config.assert_any_call("repo", "engine_sca/engine_dependencies/ConfigTool.json", "") + tool_remote.get_remote_config.assert_any_call("repo", "engine_sca/engine_dependencies/Exclusions.json", "") + # tool_remote.get_variable.assert_called_with("pipeline_name") + mock_handle_remote_config_patterns.assert_called_once() + mock_dependencies_scan.return_value.process.assert_called_once() + mock_dependencies_scan.return_value.deserializator.assert_called_once_with("scanned_dependencies") + mock_set_input_core.return_value.set_input_core.assert_called_once_with("scanned_dependencies") + + assert deserialized, ["deserialized_dependency"] + assert core_input, "core_input" + assert sbom_components is not None \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/helpers/test_get_artifacts.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/helpers/test_get_artifacts.py index 7360df638..897be5003 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/helpers/test_get_artifacts.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/helpers/test_get_artifacts.py @@ -1,9 +1,12 @@ -from devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.helpers.get_artifacts import GetArtifacts +from devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.helpers.get_artifacts import ( + GetArtifacts, +) import subprocess import pytest from unittest.mock import patch, Mock + @pytest.fixture def get_artifacts_instance(): return GetArtifacts() @@ -18,10 +21,13 @@ def test_excluded_files(get_artifacts_instance): exclusions = {"pipeline1": {"XRAY": [{"SKIP_FILES": {"files": [".py", ".txt"]}}]}} expected_result = ".js" - result = get_artifacts_instance.excluded_files(remote_config, pipeline_name, exclusions, "XRAY") + result = get_artifacts_instance.excluded_files( + remote_config, pipeline_name, exclusions, "XRAY" + ) assert result == expected_result + def test_find_packages(get_artifacts_instance): with patch("os.walk") as mock_walk, patch("os.path.join") as mock_join: working_dir = "/path/to/working_dir" @@ -38,6 +44,7 @@ def test_find_packages(get_artifacts_instance): mock_join.assert_any_call + def test_compress_and_mv_success(get_artifacts_instance): with patch("tarfile.open") as mock_tarfile_open: package = "/path/to/package" @@ -47,6 +54,7 @@ def test_compress_and_mv_success(get_artifacts_instance): mock_tarfile_open.assert_called_with(tar_path, "w") + def test_compress_and_mv_failure(get_artifacts_instance): with patch("tarfile.open") as mock_tarfile_open, patch( "os.path.basename" @@ -63,6 +71,7 @@ def test_compress_and_mv_failure(get_artifacts_instance): mock_logger_error.assert_any_call + def test_move_files(get_artifacts_instance): with patch( "devsecops_engine_tools.engine_sca.engine_dependencies.src.infrastructure.helpers.get_artifacts.logger.debug" @@ -80,6 +89,7 @@ def test_move_files(get_artifacts_instance): mock_logger_debug.assert_any_call + def test_find_artifacts(get_artifacts_instance): with patch("os.path.join") as mock_join, patch( "os.path.exists" @@ -120,4 +130,4 @@ def test_find_artifacts(get_artifacts_instance): mock_join.assert_any_call mock_compress_and_mv.assert_any_call mock_move_files.assert_called_once - mock_logger.assert_called_once \ No newline at end of file + mock_logger.assert_called_once diff --git a/tools/devsecops_engine_tools/engine_utilities/azuredevops/infrastructure/azure_devops_api.py b/tools/devsecops_engine_tools/engine_utilities/azuredevops/infrastructure/azure_devops_api.py index fb3acb63a..c5496b156 100644 --- a/tools/devsecops_engine_tools/engine_utilities/azuredevops/infrastructure/azure_devops_api.py +++ b/tools/devsecops_engine_tools/engine_utilities/azuredevops/infrastructure/azure_devops_api.py @@ -2,6 +2,7 @@ from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError from urllib.parse import urlsplit, unquote from azure.devops.connection import Connection +from azure.devops.v7_1.wiki.models import GitVersionDescriptor from msrest.authentication import BasicAuthentication from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities.settings import SETTING_LOGGER @@ -48,14 +49,19 @@ def get_azure_connection(self) -> Connection: except Exception as e: raise ApiError("Error getting Azure DevOps connection: " + str(e)) - def get_remote_json_config(self, connection: Connection): + def get_remote_json_config(self, connection: Connection, branch=""): try: git_client = connection.clients.get_git_client() + version_descriptor = None + if branch: version_descriptor = GitVersionDescriptor(version=branch, version_type="branch") + file_content = git_client.get_item_text( repository_id=self.__repository_id, path=self.__remote_config_path, project=self.__project_remote_config, + version_descriptor=version_descriptor ) + data = json.loads(b"".join(file_content).decode("utf-8")) return data except Exception as e: diff --git a/tools/devsecops_engine_tools/engine_utilities/azuredevops/models/AzurePredefinedVariables.py b/tools/devsecops_engine_tools/engine_utilities/azuredevops/models/AzurePredefinedVariables.py index 5248a4cfe..92d3db89c 100644 --- a/tools/devsecops_engine_tools/engine_utilities/azuredevops/models/AzurePredefinedVariables.py +++ b/tools/devsecops_engine_tools/engine_utilities/azuredevops/models/AzurePredefinedVariables.py @@ -63,3 +63,8 @@ class AgentVariables(BaseEnum): Agent_WorkFolder = "Agent.WorkFolder" Agent_TempDirectory = "Agent.TempDirectory" Agent_OS = "Agent.OS" + +class VMVariables(BaseEnum): + Vm_Product_Type_Name = "Vm.Product.Type.Name" + Vm_Product_Name = "Vm.Product.Name" + Vm_Product_Description = "Vm.Product.Description" diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/__init__.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/__init__.py index 202ecefbf..5f37a2de2 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/__init__.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/__init__.py @@ -3,4 +3,7 @@ from .applications.defect_dojo import DefectDojo from .applications.finding import Finding from .applications.connect import Connect -from .applications.engagement import Engagement \ No newline at end of file +from .applications.engagement import Engagement +from .applications.product import Product +from .applications.component import Component +from .applications.finding_exclusion import FindingExclusion \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/component.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/component.py new file mode 100644 index 000000000..434a45cec --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/component.py @@ -0,0 +1,28 @@ +from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.component import ComponentRestConsumer +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.user_case.component import ComponentUserCase +from devsecops_engine_tools.engine_utilities.settings import SETTING_LOGGER + +logger = MyLogger.__call__(**SETTING_LOGGER).get_logger() + +class Component: + + @staticmethod + def get_component(session, request: dict): + try: + rest_component = ComponentRestConsumer(session=session) + uc = ComponentUserCase(rest_component) + return uc.get(request) + except ApiError as e: + logger.error(f"Error during get component: {e}") + raise e + + @staticmethod + def create_component(session, request): + try: + rest_component = ComponentRestConsumer(session=session) + uc = ComponentUserCase(rest_component) + return uc.post(request) + except ApiError as e: + raise e diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/connect.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/connect.py index 25eb61da7..363402c8a 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/connect.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/connect.py @@ -35,3 +35,7 @@ def cmdb(**kwargs) -> ImportScanRequest: return e return response + + def get_code_app(engagement_name, expression): + uc = CmdbUserCase(rest_consumer_cmdb=None, utils_azure=None, expression=expression) + return uc.get_code_app(engagement_name) diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/defect_dojo.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/defect_dojo.py index 9b2ea6cc1..a288a85ed 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/defect_dojo.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/defect_dojo.py @@ -1,14 +1,26 @@ from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger -from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.import_scan import ImportScanRestConsumer -from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.product_type import ProductTypeRestConsumer -from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.product import ProductRestConsumer +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.import_scan import ( + ImportScanRestConsumer, +) +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.product_type import ( + ProductTypeRestConsumer, +) +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.product import ( + ProductRestConsumer, +) from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.scan_configurations import ( ScanConfigrationRestConsumer, ) -from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.engagement import EngagementRestConsumer -from devsecops_engine_tools.engine_utilities.defect_dojo.domain.request_objects.import_scan import ImportScanRequest -from devsecops_engine_tools.engine_utilities.defect_dojo.domain.user_case.import_scan import ImportScanUserCase +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.engagement import ( + EngagementRestConsumer, +) +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.request_objects.import_scan import ( + ImportScanRequest, +) +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.user_case.import_scan import ( + ImportScanUserCase, +) from devsecops_engine_tools.engine_utilities.utils.session_manager import SessionManager from devsecops_engine_tools.engine_utilities.settings import SETTING_LOGGER @@ -22,11 +34,20 @@ def send_import_scan(request: ImportScanRequest): if not isinstance(request, ImportScanRequest): return request rest_import_scan = ImportScanRestConsumer(request, session=SessionManager()) - rest_product_type = ProductTypeRestConsumer(request, session=SessionManager()) - rest_product = ProductRestConsumer(request, session=SessionManager()) + rest_product_type = ProductTypeRestConsumer( + request, session=SessionManager() + ) + rest_product = ProductRestConsumer( + SessionManager( + request.token_defect_dojo, + request.host_defect_dojo, + ) + ) rest_engagement = EngagementRestConsumer(request, session=SessionManager()) - rest_scan_configuration = ScanConfigrationRestConsumer(request, session=SessionManager()) + rest_scan_configuration = ScanConfigrationRestConsumer( + request, session=SessionManager() + ) uc = ImportScanUserCase( rest_import_scan, rest_product_type, @@ -36,4 +57,5 @@ def send_import_scan(request: ImportScanRequest): ) return uc.execute(request) except ApiError as e: + logger.error(f"Error during import scan: {e}") raise e diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/finding.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/finding.py index 065dd841c..fa6973b01 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/finding.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/finding.py @@ -1,9 +1,6 @@ -from devsecops_engine_tools.engine_utilities.defect_dojo.domain.request_objects.finding import FindingRequest from devsecops_engine_tools.engine_utilities.defect_dojo.domain.serializers.finding import FindingSerializer from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.finding import FindingRestConsumer from devsecops_engine_tools.engine_utilities.defect_dojo.domain.user_case.finding import FindingUserCase, FindingGetUserCase -from devsecops_engine_tools.engine_utilities.utils.session_manager import SessionManager -from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/finding_exclusion.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/finding_exclusion.py new file mode 100644 index 000000000..37300dbe2 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/finding_exclusion.py @@ -0,0 +1,14 @@ +from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.user_case.finding_exclusion import FindingExclusionUserCase +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.finding_exclusion import FindingExclusionRestConsumer + +class FindingExclusion: + @staticmethod + def get_finding_exclusion(session, **request): + try: + rest_finding_exclusion = FindingExclusionRestConsumer(session=session) + + uc = FindingExclusionUserCase(rest_finding_exclusion) + return uc.execute(request) + except ApiError as e: + raise e \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/product.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/product.py new file mode 100644 index 000000000..1c9fa1002 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/applications/product.py @@ -0,0 +1,14 @@ +from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.user_case.product import ProductUserCase +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.product import ProductRestConsumer + +class Product: + @staticmethod + def get_product(session, request: dict): + try: + rest_product = ProductRestConsumer(session=session) + + uc = ProductUserCase(rest_product) + return uc.execute(request) + except ApiError as e: + raise e \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/component.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/component.py new file mode 100644 index 000000000..c38d30fb7 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/component.py @@ -0,0 +1,20 @@ +import dataclasses +from typing import List +from devsecops_engine_tools.engine_utilities.utils.dataclass_classmethod import FromDictMixin + + +@dataclasses.dataclass +class Component(FromDictMixin): + id: int = 0 + name: str = "" + version: str = "" + date: str = "" + Component_id: int = 0 + + +@dataclasses.dataclass +class ComponentList(FromDictMixin): + count: int = 0 + next = None + previous = None + results: List[Component] = dataclasses.field(default_factory=list) diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/engagement.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/engagement.py index 6e6c05c1a..9375a45bb 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/engagement.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/engagement.py @@ -41,6 +41,7 @@ class Engagement(FromDictMixin): build_server: str = "" source_code_management_server: str = "" orchestration_engine: str = "" + vm_url: str = "" notes = [] files = [] risk_acceptance = [] diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/finding_exclusion.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/finding_exclusion.py new file mode 100644 index 000000000..b19e72cc1 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/finding_exclusion.py @@ -0,0 +1,20 @@ +import dataclasses +from typing import List +from devsecops_engine_tools.engine_utilities.utils.dataclass_classmethod import FromDictMixin + + +@dataclasses.dataclass +class FindingExclusion(FromDictMixin): + uuid: str = "" + unique_id_from_tool: str = "" + type: str = "" + create_date: str = "" + expiration_date: str = "" + + +@dataclasses.dataclass +class FindingExclusionList(FromDictMixin): + count: int = 0 + next = None + previous = None + results: List[FindingExclusion] = dataclasses.field(default_factory=list) diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/product_list.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/product_list.py index aed0b3f92..e2a5884f6 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/product_list.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/models/product_list.py @@ -1,12 +1,18 @@ import dataclasses -from typing import List +from typing import List, Dict from devsecops_engine_tools.engine_utilities.utils.dataclass_classmethod import FromDictMixin from devsecops_engine_tools.engine_utilities.defect_dojo.domain.models.product import Product +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.models.product_type import ProductType +@dataclasses.dataclass +class Prefetch(FromDictMixin): + prod_type: Dict[str, ProductType] + @dataclasses.dataclass class ProductList(FromDictMixin): count: int = 0 next = None previous = None results: List[Product] = dataclasses.field(default_factory=list) + prefetch: Prefetch = None diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/request_objects/import_scan.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/request_objects/import_scan.py index 2f4198bc8..25b4372fc 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/request_objects/import_scan.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/request_objects/import_scan.py @@ -40,6 +40,7 @@ class ImportScanRequest: code_app: str = "" token_cmdb: str = "" host_cmdb: str = "" + cmdb_request_response: dict = None token_defect_dojo: str = "" host_defect_dojo: str = "" # *** config map *** diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/serializers/finding.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/serializers/finding.py index c36f03062..1b3ba91ca 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/serializers/finding.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/serializers/finding.py @@ -63,7 +63,7 @@ class FindingSerializer(Schema): reviewers = fields.List(fields.Int, requerided=False) risk_accetance = fields.Int(requerided=False) risk_status = fields.Str( - required=False, validate=validate.OneOf(["Risk Pending", "Risk Rejected", "Risk Expired", "Risk Accepted", "Risk Active", "Transfer Pending", "Transfer Rejected", "Transfer Expired", "Transfer Accepted"]) + required=False, validate=validate.OneOf(["Risk Pending", "Risk Rejected", "Risk Expired", "Risk Accepted", "Risk Active", "Transfer Pending", "Transfer Rejected", "Transfer Expired", "Transfer Accepted", "On Whitelist", "On Blacklist"]) ) risk_accepted = fields.Bool(requerided=False) sast_sink_object = fields.Str(requeride=False) diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/serializers/import_scan.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/serializers/import_scan.py index c00175692..8a953e1cf 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/serializers/import_scan.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/serializers/import_scan.py @@ -198,16 +198,17 @@ class ImportScanSerializer(Schema): service = fields.Str(required=False) group_by = fields.Str(required=False) test_title = fields.Str(required=False) - description_product = fields.Str(required=False) + product_description = fields.Str(required=False) create_finding_groups_for_all_findings = fields.Str(required=False) tools_configuration = fields.Int(required=False, load_default=1) code_app = fields.Str(required=False) # defect-dojo credential - token_cmdb = fields.Str(required=True) - host_cmdb = fields.Url(required=True) + token_cmdb = fields.Str(required=False) + host_cmdb = fields.Url(required=False) + cmdb_request_response = fields.Dict(required=False) token_defect_dojo = fields.Str(required=True) host_defect_dojo = fields.Str(required=True) - cmdb_mapping = fields.Dict(required=True) + cmdb_mapping = fields.Dict(required=False) product_type_name_mapping = fields.Dict(required=False) # Config remote credential compact_remote_config_url = fields.Str(required=False) diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/component.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/component.py new file mode 100644 index 000000000..d873c1418 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/component.py @@ -0,0 +1,11 @@ +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.component import ComponentRestConsumer + +class ComponentUserCase: + def __init__(self, rest_component: ComponentRestConsumer): + self.__rest_component = rest_component + + def get(self, request): + return self.__rest_component.get_component(request) + + def post(self, request): + return self.__rest_component.post_component(request) diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/finding_exclusion.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/finding_exclusion.py new file mode 100644 index 000000000..491c723fd --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/finding_exclusion.py @@ -0,0 +1,9 @@ +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.finding_exclusion import FindingExclusionRestConsumer + +class FindingExclusionUserCase: + def __init__(self, rest_finding_exclusion: FindingExclusionRestConsumer): + self.__rest_finding_exclusion = rest_finding_exclusion + + def execute(self, request): + response = self.__rest_finding_exclusion.get_finding_exclusions(request) + return response diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/import_scan.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/import_scan.py index a40af4b84..c1aa4783f 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/import_scan.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/import_scan.py @@ -43,7 +43,7 @@ def execute(self, request: ImportScanRequest) -> ImportScanRequest: raise ApiError(log) logger.info(f"Match {request.scan_type}") - products = self.__rest_product.get_products(request) + products = self.__rest_product.get_products({"name":request.code_app}) if len(products.results) > 0: product_id = products.results[0].id request.product_name = products.results[0].name @@ -66,12 +66,12 @@ def execute(self, request: ImportScanRequest) -> ImportScanRequest: with id {product_type_id}" ) - product = self.__rest_product.post_product(request, product_type_id) - product_id = product.id - logger.info( - f"product created: {product.name}\ - found with id: {product.id}" - ) + product = self.__rest_product.post_product(request, product_type_id) + product_id = product.id + logger.info( + f"product created: {product.name}\ + found with id: {product.id}" + ) api_scan_bool = re.search(" API ", request.scan_type) if api_scan_bool: diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/product.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/product.py new file mode 100644 index 000000000..67c06be4b --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/domain/user_case/product.py @@ -0,0 +1,9 @@ +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.product import ProductRestConsumer + +class ProductUserCase: + def __init__(self, rest_product: ProductRestConsumer): + self.__rest_product = rest_product + + def execute(self, request): + response = self.__rest_product.get_products(request) + return response diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/cmdb.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/cmdb.py index 38a7519e0..1892d6dea 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/cmdb.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/cmdb.py @@ -1,4 +1,5 @@ import json +import ast from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities.defect_dojo.domain.models.cmdb import Cmdb @@ -18,39 +19,89 @@ def __init__(self, token: str, host: str, mapping_cmdb: dict, session: SessionMa self.__session = session._instance def get_product_info(self, request: ImportScanRequest) -> Cmdb: - data = json.dumps({"codapp": request.code_app}) - headers = {"tokenkey": self.__token, "Content-Type": "application/json"} - logger.info("Search info of name product") - cmdb_object = Cmdb( - product_type_name="ORPHAN_PRODUCT_TYPE", - product_name=f"{request.code_app}_Product", - tag_product="ORPHAN", - product_description="Orphan Product Description", - codigo_app=str(request.code_app), - ) + method = request.cmdb_request_response.get("METHOD") + headers = self.prepare_headers(request.cmdb_request_response.get("HEADERS")) + response_format = request.cmdb_request_response.get("RESPONSE") + + if method not in ["GET", "POST"]: + raise ValueError(f"Unsupported method: {method}") + + return self.handle_request(method, headers, request, response_format) + + def handle_request(self, method, headers, request: ImportScanRequest, response_format) -> Cmdb: + cmdb_object = self.initialize_cmdb_object(request) + try: - response = self.__session.post(self.__host, headers=headers, data=data, verify=VERIFY_CERTIFICATE) - if response.status_code != 200: - logger.warning(response) - raise ApiError(f"Error querying cmdb: {response.reason}") - - if response.json() == []: - e = f"Engagement: {request.code_app} not found" - logger.warning(e) - # Producto is Orphan - return cmdb_object - - data = response.json()[-1] - data_map = self.mapping_cmdb(data) - logger.info(data_map) - cmdb_object = Cmdb.from_dict(data_map) + if method == "GET": + params = self.replace_placeholders( + request.cmdb_request_response.get("PARAMS", {}), + request.code_app + ) + response = self.__session.get(self.__host, headers=headers, params=params, verify=VERIFY_CERTIFICATE) + elif method == "POST": + body = self.replace_placeholders( + request.cmdb_request_response.get("BODY", {}), + request.code_app + ) + body_json = json.dumps(body) + response = self.__session.post(self.__host, headers=headers, data=body_json, verify=VERIFY_CERTIFICATE) + + return self.process_response(response, response_format, cmdb_object, request.code_app) except Exception as e: logger.warning(e) return cmdb_object + + def process_response(self, response, response_format, cmdb_object, code_app) -> Cmdb: + if response.status_code != 200: + logger.warning(response) + raise ApiError(f"Error querying cmdb: {response.reason}") + + if response.json() == []: + logger.warning(f"Engagement: {code_app} not found") + return cmdb_object # Producto es Orphan + + data = self.get_nested_data(response, response_format) + data_map = self.mapping_cmdb(data) + logger.info(data_map) + cmdb_object = Cmdb.from_dict(data_map) + cmdb_object.codigo_app = code_app return cmdb_object - def mapping_cmdb(self, data): - data_map = {} - for key, value in self.__mapping_cmdb.items(): - data_map[key] = data[value] if value in data else "" - return data_map \ No newline at end of file + def initialize_cmdb_object(self, request: ImportScanRequest) -> Cmdb: + return Cmdb( + product_type_name="ORPHAN_PRODUCT_TYPE", + product_name=f"{request.code_app}_Product", + tag_product="ORPHAN", + product_description="Orphan Product Description", + codigo_app=str(request.code_app), + ) + + def mapping_cmdb(self, data: dict) -> dict: + return {key: data.get(value, "") for key, value in self.__mapping_cmdb.items()} + + def get_nested_data(self, response, keys: list) -> dict: + data = response.json() + for key in keys: + if isinstance(data, dict) and key in data: + data = data[key] + elif isinstance(data, list) and isinstance(key, int): + key = key if key >=0 else len(data) + key + if 0 <= key < len(data): + data = data[key] + else: + raise KeyError(f"Index '{key}' out of range in the current context.") + else: + raise KeyError(f"Key '{key}' not found or invalid in the current context.") + return data + + def prepare_headers(self, headers: dict) -> dict: + return {key: (self.__token if value == 'tokenvalue' else value) for key, value in headers.items()} + + def replace_placeholders(self, data, replacements): + data = str(data) + data = data.replace("codappvalue", replacements) + try: + return ast.literal_eval(data) + except (SyntaxError, ValueError) as e: + raise ValueError(f"Error converting string to dictionary: {e}") + diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/component.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/component.py new file mode 100644 index 000000000..461eb839c --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/component.py @@ -0,0 +1,52 @@ +import json +from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.settings.settings import VERIFY_CERTIFICATE +from devsecops_engine_tools.engine_utilities.utils.session_manager import SessionManager +from devsecops_engine_tools.engine_utilities.settings import SETTING_LOGGER +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger + +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.models.component import ComponentList, Component + +logger = MyLogger.__call__(**SETTING_LOGGER).get_logger() + + +class ComponentRestConsumer: + def __init__(self, session: SessionManager): + self.__token = session._token + self.__host = session._host + self.__session = session._instance + + def get_component(self, request): + url = f"{self.__host}/api/v2/components/" + headers = { + "Authorization": f"Token {self.__token}", + "Content-Type": "application/json", + } + try: + response = self.__session.get( + url=url, headers=headers, params=request, verify=VERIFY_CERTIFICATE + ) + if response.status_code != 200: + logger.error(response.json()) + raise ApiError(response.json()) + components = ComponentList().from_dict(response.json()) + except Exception as e: + raise ApiError(e) + return components + + def post_component(self, request): + url = f"{self.__host}/api/v2/components/" + headers = { + "Authorization": f"Token {self.__token}", + "Content-Type": "application/json", + } + try: + response = self.__session.post( + url=url, headers=headers, data=json.dumps(request), verify=VERIFY_CERTIFICATE + ) + if response.status_code != 201: + raise ApiError(response.json()) + response = Component.from_dict(response.json()) + except Exception as e: + raise ApiError(e) + return response diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/finding_exclusion.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/finding_exclusion.py new file mode 100644 index 000000000..eb7c94cbd --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/finding_exclusion.py @@ -0,0 +1,28 @@ +from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.models.finding_exclusion import FindingExclusionList +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.settings.settings import VERIFY_CERTIFICATE +from devsecops_engine_tools.engine_utilities.utils.session_manager import SessionManager +from devsecops_engine_tools.engine_utilities.settings import SETTING_LOGGER + +logger = MyLogger.__call__(**SETTING_LOGGER).get_logger() + +class FindingExclusionRestConsumer: + def __init__(self, session: SessionManager): + self.__token = session._token + self.__host = session._host + self.__session = session._instance + + + def get_finding_exclusions(self, request) -> FindingExclusionList: + url = f"{self.__host}/api/v2/finding_exclusions/" + headers = {"Authorization": f"Token {self.__token}", "Content-Type": "application/json"} + try: + response = self.__session.get(url, headers=headers, params=request, verify=VERIFY_CERTIFICATE) + if response.status_code != 200: + raise ApiError(response.json()) + finding_exclusions_object = FindingExclusionList.from_dict(response.json()) + except Exception as e: + logger.error(f"from dict FindingExclusion: {e}") + raise ApiError(e) + return finding_exclusions_object diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/product.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/product.py index 061c00b4b..7cada97b7 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/product.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/infraestructure/driver_adapters/product.py @@ -11,16 +11,17 @@ class ProductRestConsumer: - def __init__(self, request: ImportScanRequest, session: SessionManager): - self.__token = request.token_defect_dojo - self.__host = request.host_defect_dojo + def __init__(self, session: SessionManager): + self.__token = session._token + self.__host = session._host self.__session = session._instance - def get_products(self, request: ImportScanRequest) -> ProductList: - url = f"{self.__host}/api/v2/products/?name={request.code_app}" + + def get_products(self, request) -> ProductList: + url = f"{self.__host}/api/v2/products/" headers = {"Authorization": f"Token {self.__token}", "Content-Type": "application/json"} try: - response = self.__session.get(url, headers=headers, data={}, verify=VERIFY_CERTIFICATE) + response = self.__session.get(url, headers=headers, params=request, verify=VERIFY_CERTIFICATE) if response.status_code != 200: raise ApiError(response.json()) products_object = ProductList.from_dict(response.json()) diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/applications/test_component.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/applications/test_component.py new file mode 100644 index 000000000..1ede4f205 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/applications/test_component.py @@ -0,0 +1,53 @@ +import unittest +from unittest.mock import patch, MagicMock +from devsecops_engine_tools.engine_utilities.defect_dojo.applications.component import ( + Component, +) +from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError + + +class TestComponent(unittest.TestCase): + + @patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.applications.component.ComponentRestConsumer" + ) + def test_get_components(self, mock_rest_consumer): + mock_rest_consumer.return_value.get_component.return_value = "response" + session = MagicMock() + request = MagicMock() + assert Component.get_component(session, request) == "response" + + @patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.applications.component.ComponentRestConsumer" + ) + def test_get_components_raises_api_error(self, mock_component_rest_consumer): + mock_component_rest_consumer.return_value.get_component.side_effect = Exception( + "error" + ) + session = MagicMock() + request = MagicMock() + with self.assertRaises(Exception) as e: + Component.get_component(session, request) + assert str(e) == "error" + + @patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.applications.component.ComponentRestConsumer" + ) + def test_create_component(self, mock_rest_consumer): + mock_rest_consumer.return_value.post_component.return_value = "response" + session = MagicMock() + request = MagicMock() + assert Component.create_component(session, request) == "response" + + @patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.applications.component.ComponentRestConsumer" + ) + def test_create_components_raises_api_error(self, mock_component_rest_consumer): + mock_component_rest_consumer.return_value.post_component.side_effect = Exception( + "error" + ) + session = MagicMock() + request = MagicMock() + with self.assertRaises(Exception) as e: + Component.create_component(session, request) + assert str(e) == "error" \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/applications/test_finding_exclusion.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/applications/test_finding_exclusion.py new file mode 100644 index 000000000..08bcff3dc --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/applications/test_finding_exclusion.py @@ -0,0 +1,36 @@ +import unittest +from unittest.mock import patch, Mock +from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError +from devsecops_engine_tools.engine_utilities.defect_dojo.applications.finding_exclusion import FindingExclusion + +class TestFindingExclusion(unittest.TestCase): + + @patch('devsecops_engine_tools.engine_utilities.defect_dojo.applications.finding_exclusion.FindingExclusionRestConsumer') + @patch('devsecops_engine_tools.engine_utilities.defect_dojo.applications.finding_exclusion.FindingExclusionUserCase') + def test_get_finding_exclusion_success(self, mock_user_case, mock_rest_consumer): + session = Mock() + request = {'key': 'value'} + mock_uc_instance = mock_user_case.return_value + mock_uc_instance.execute.return_value = 'expected_result' + + result = FindingExclusion.get_finding_exclusion(session, **request) + + mock_rest_consumer.assert_called_once_with(session=session) + mock_user_case.assert_called_once_with(mock_rest_consumer.return_value) + mock_uc_instance.execute.assert_called_once_with(request) + self.assertEqual(result, 'expected_result') + + @patch('devsecops_engine_tools.engine_utilities.defect_dojo.applications.finding_exclusion.FindingExclusionRestConsumer') + @patch('devsecops_engine_tools.engine_utilities.defect_dojo.applications.finding_exclusion.FindingExclusionUserCase') + def test_get_finding_exclusion_api_error(self, mock_user_case, mock_rest_consumer): + session = Mock() + request = {'key': 'value'} + mock_uc_instance = mock_user_case.return_value + mock_uc_instance.execute.side_effect = ApiError('API error occurred') + + with self.assertRaises(ApiError): + FindingExclusion.get_finding_exclusion(session, **request) + + mock_rest_consumer.assert_called_once_with(session=session) + mock_user_case.assert_called_once_with(mock_rest_consumer.return_value) + mock_uc_instance.execute.assert_called_once_with(request) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/applications/test_product.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/applications/test_product.py new file mode 100644 index 000000000..3fdf2cab2 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/applications/test_product.py @@ -0,0 +1,36 @@ +from unittest.mock import MagicMock, patch + +from devsecops_engine_tools.engine_utilities.defect_dojo.applications.product import ( + Product, +) + + + +@patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.applications.product.ProductRestConsumer" +) +def test_get_products(mock_product_rest_consumer): + mock_product_rest_consumer.return_value.get_products.return_value = ( + "response" + ) + session = MagicMock() + request = MagicMock() + assert Product.get_product(session, request) == "response" + + + +@patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.applications.product.ProductRestConsumer" +) +def test_get_products_raises_api_error( + mock_product_rest_consumer +): + mock_product_rest_consumer.return_value.get_products.side_effect = Exception( + "error" + ) + session = MagicMock() + request = MagicMock() + try: + Product.get_product(session, request) + except Exception as e: + assert str(e) == "error" diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_connect_cmdb.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_connect_cmdb.py index b93814fab..e2bd2378d 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_connect_cmdb.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_connect_cmdb.py @@ -84,6 +84,17 @@ def test_execute(engagement_name, obj_cmdb): "project_remote_config": "Vicepresidencia Servicios de Tecnología", "token_cmdb": "123456789", "host_cmdb": "http://localhost:8000", + "cmdb_request_response": { + "HEADERS": { + "Content-Type": "application/json", + "tokenkey": "tokenvalue" + }, + "METHOD": "POST", + "BODY": { + "codapp": "codappvalue" + }, + "RESPONSE": [0] + }, "expression": "((AUD|AP|CLD|USR|OPS|ASN|AW|NU|EUC|IS))_", "token_defect_dojo": "123456789101212", "host_defect_dojo": "http://localhost:8000", diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_finding_exclusion.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_finding_exclusion.py new file mode 100644 index 000000000..b9645093b --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_finding_exclusion.py @@ -0,0 +1,32 @@ +import unittest +from unittest.mock import MagicMock +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.user_case.finding_exclusion import FindingExclusionUserCase +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.finding_exclusion import FindingExclusionRestConsumer + +class TestFindingExclusionUserCase(unittest.TestCase): + def setUp(self): + self.mock_rest_finding_exclusion = MagicMock(spec=FindingExclusionRestConsumer) + self.user_case = FindingExclusionUserCase(self.mock_rest_finding_exclusion) + + def test_execute_success(self): + request = {"some": "data"} + expected_response = {"response": "data"} + self.mock_rest_finding_exclusion.get_finding_exclusions.return_value = expected_response + + response = self.user_case.execute(request) + + self.assertEqual(response, expected_response) + self.mock_rest_finding_exclusion.get_finding_exclusions.assert_called_once_with(request) + + def test_execute_no_data(self): + request = {} + expected_response = {"response": "no_data"} + self.mock_rest_finding_exclusion.get_finding_exclusions.return_value = expected_response + + response = self.user_case.execute(request) + + self.assertEqual(response, expected_response) + self.mock_rest_finding_exclusion.get_finding_exclusions.assert_called_once_with(request) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_import_scan.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_import_scan.py index 06eaf7d2a..98c9572ba 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_import_scan.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_import_scan.py @@ -42,7 +42,7 @@ def test_user_case_creation(): assert isinstance(request, ImportScanRequest) rest_import_scan = ImportScanRestConsumer(request, SessionManager()) rest_product_type = ProductTypeRestConsumer(request, SessionManager()) - rest_product = ProductRestConsumer(request, SessionManager()) + rest_product = ProductRestConsumer(SessionManager()) rest_scan_configuration = ScanConfigrationRestConsumer(request, SessionManager()) rest_engagement = EngagementRestConsumer(request, SessionManager()) uc = ImportScanUserCase( diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_product.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_product.py new file mode 100644 index 000000000..86304d9a0 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/domain/user_case/test_product.py @@ -0,0 +1,18 @@ +from unittest.mock import Mock +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.user_case.product import ProductUserCase +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.product import ProductRestConsumer +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.models.product import Product +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.models.product_list import ProductList +from requests import Response + + +def test_execute_product_get(): + mock_rest_product = Mock() + # Creation mocks, get and close + mock_rest_product.get_products.return_value = ProductList(count=1, results=[Product(id=1), Product(id=2)]) + response = Response() + response.status_code = 200 + mock_rest_product.get_products.return_value = response + uc = ProductUserCase(mock_rest_product) + response = uc.execute({"codeapp": "name"}) + assert response.status_code == 200 diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_cmdb_rc.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_cmdb_rc.py index 563c52fab..07ae88477 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_cmdb_rc.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_cmdb_rc.py @@ -1,4 +1,5 @@ import pytest +import json from unittest.mock import Mock from devsecops_engine_tools.engine_utilities.defect_dojo.domain.models.cmdb import Cmdb from devsecops_engine_tools.engine_utilities.defect_dojo.test.files.get_response import session_manager_post @@ -10,7 +11,11 @@ def test_get_product_info_success(): request = ImportScanRequest() request.code_app = "123" - request.product_name = "test_product_name" + request.cmdb_request_response = { + "METHOD": "POST", + "HEADERS": {"Content-Type": "application/json"}, + "RESPONSE": [0] + } session_mock = session_manager_post( status_code=200, mock_response=[{"name_cmdb": "NU1245_Test", "product_type_name_cmdb": "software"}] ) @@ -34,7 +39,10 @@ def test_get_product_info_success(): def test_get_product_info_failure(): request = ImportScanRequest() request.code_app = "123" - request.product_name = "test_product_name" + request.cmdb_request_response = { + "METHOD": "POST", + "HEADERS": {"Content-Type": "application/json"} + } session_mock = session_manager_post(status_code=500, mock_response={"Message": "Error mock"}) consumer = CmdbRestConsumer( "token12345", @@ -45,3 +53,161 @@ def test_get_product_info_failure(): response = consumer.get_product_info(request) assert response.product_type_name == "ORPHAN_PRODUCT_TYPE" + + +def test_get_product_info_unsupported_method(): + request = ImportScanRequest() + request.code_app = "123" + request.cmdb_request_response = { + "METHOD": "PUT", + "HEADERS": {"Content-Type": "application/json"} + } + session_mock = session_manager_post(status_code=500, mock_response={"Message": "Error mock"}) + consumer = CmdbRestConsumer( + "token12345", + "http://hosttest.com", + {"product_name": "name_cmdb", "product_type_name": "product_type_name_cmdb"}, + session_mock, + ) + + with pytest.raises(ValueError): + consumer.get_product_info(request) + + +def test_process_response_success(): + response = Mock() + response.status_code = 200 + response.json.return_value = [{"name_cmdb": "NU1245_Test", "product_type_name_cmdb": "software"}] + consumer = CmdbRestConsumer( + "token12345", + "http://hosttest.com", + {"product_name": "name_cmdb", "product_type_name": "product_type_name_cmdb"}, + Mock(), + ) + + cmdb_object = consumer.process_response(response, [0], Cmdb(), "123") + + assert isinstance(cmdb_object, Cmdb) + assert cmdb_object.product_name == "NU1245_Test" + assert cmdb_object.product_type_name == "software" + + +def test_initialize_cmdb_object(): + consumer = CmdbRestConsumer( + "token12345", + "http://hosttest.com", + {"product_name": "name_cmdb", "product_type_name": "product_type_name_cmdb"}, + Mock(), + ) + + request = ImportScanRequest(code_app="TEST123") + + cmdb_object = consumer.initialize_cmdb_object(request) + + expected_cmdb = Cmdb( + product_type_name="ORPHAN_PRODUCT_TYPE", + product_name="TEST123_Product", + tag_product="ORPHAN", + product_description="Orphan Product Description", + codigo_app="TEST123", + ) + assert cmdb_object == expected_cmdb + + +def test_mapping_cmdb_valid_data(): + consumer = CmdbRestConsumer( + "token12345", + "http://hosttest.com", + {"product_name": "name_cmdb", "product_type_name": "product_type_name_cmdb"}, + Mock(), + ) + + data = {"name_cmdb": "NU1245_Test", "product_type_name_cmdb": "software"} + + result = consumer.mapping_cmdb(data) + + expected = {"product_name": "NU1245_Test", "product_type_name": "software"} + assert result == expected + + +def test_mapping_cmdb_missing_data(): + consumer = CmdbRestConsumer( + "token12345", + "http://hosttest.com", + {"product_name": "name_cmdb", "product_type_name": "product_type_name_cmdb"}, + Mock(), + ) + + data = {"name_cmdb": "NU1245_Test"} + + result = consumer.mapping_cmdb(data) + + expected = {"product_name": "NU1245_Test", "product_type_name": ""} + assert result == expected + + +def test_get_nested_data_with_valid_keys(): + consumer = CmdbRestConsumer( + "token12345", + "http://hosttest.com", + {"product_name": "name_cmdb", "product_type_name": "product_type_name_cmdb"}, + Mock(), + ) + response_mock = Mock() + response_mock.json.return_value = {"name_cmdb": "NU1245_Test", "product_type_name_cmdb": "software"} + keys = ["name_cmdb"] + + result = consumer.get_nested_data(response_mock, keys) + + assert result == "NU1245_Test" + + +def test_get_nested_data_with_invalid_keys(): + consumer = CmdbRestConsumer( + "token12345", + "http://hosttest.com", + {"product_name": "name_cmdb", "product_type_name": "product_type_name_cmdb"}, + Mock(), + ) + + response_mock = Mock() + response_mock.json.return_value = {"name_cmdb": "NU1245_Test", "product_type_name_cmdb": "software"} + keys = ["name_cmdb", "product_type_name_cmdb", "invalid_key"] + + with pytest.raises(KeyError): + consumer.get_nested_data(response_mock, keys) + + +def test_prepare_headers_with_tokenvalue(): + consumer = CmdbRestConsumer( + "token12345", + "http://hosttest.com", + {"product_name": "name_cmdb", "product_type_name": "product_type_name_cmdb"}, + Mock(), + ) + + headers = {"Authorization": "tokenvalue", "Content-Type": "application/json"} + + result = consumer.prepare_headers(headers) + + expected = {"Authorization": "token12345", "Content-Type": "application/json"} + assert result == expected + + +def test_replace_placeholders_valid_replacement(): + consumer = CmdbRestConsumer( + "token12345", + "http://hosttest.com", + {"product_name": "name_cmdb", "product_type_name": "product_type_name_cmdb"}, + Mock(), + ) + + data = '{"key": "codappvalue"}' + replacements = "new_value" + + # Act + result = consumer.replace_placeholders(data, replacements) + + # Assert + expected = {"key": "new_value"} + assert result == expected \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_component.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_component.py new file mode 100644 index 000000000..5560169d0 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_component.py @@ -0,0 +1,158 @@ +import unittest +import json +from unittest.mock import patch, MagicMock +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.component import ( + ComponentRestConsumer, +) +from devsecops_engine_tools.engine_core.src.domain.model.component import Component +from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError +from devsecops_engine_tools.engine_utilities.utils.session_manager import SessionManager +from devsecops_engine_tools.engine_utilities.defect_dojo.domain.models.component import ( + ComponentList, +) + + +class TestComponentRestConsumer(unittest.TestCase): + def setUp(self): + self.session = MagicMock(spec=SessionManager) + self.session._token = "fake_token" + self.session._host = "http://fakehost" + self.session._instance = MagicMock() + self.consumer = ComponentRestConsumer(self.session) + + @patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.component.VERIFY_CERTIFICATE", + True, + ) + @patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.component.ComponentList" + ) + def test_get_component_success(self, mock_component_list): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"components": []} + self.session._instance.get.return_value = mock_response + mock_component_list().from_dict.return_value = ComponentList() + + request = {"param": "value"} + components = self.consumer.get_component(request) + + self.session._instance.get.assert_called_once_with( + url="http://fakehost/api/v2/components/", + headers={ + "Authorization": "Token fake_token", + "Content-Type": "application/json", + }, + params=request, + verify=True, + ) + mock_component_list().from_dict.assert_called_once_with({"components": []}) + self.assertIsInstance(components, ComponentList) + + @patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.component.logger" + ) + def test_get_component_failure(self, mock_logger): + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.json.return_value = {"error": "Bad Request"} + self.session._instance.get.return_value = mock_response + + request = {"param": "value"} + with self.assertRaises(ApiError): + self.consumer.get_component(request) + + self.session._instance.get.assert_called_once_with( + url="http://fakehost/api/v2/components/", + headers={ + "Authorization": "Token fake_token", + "Content-Type": "application/json", + }, + params=request, + verify=False, + ) + mock_logger.error.assert_called_once_with({"error": "Bad Request"}) + + @patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.component.logger" + ) + def test_get_component_exception(self, mock_logger): + self.session._instance.get.side_effect = Exception("Some error") + + request = {"param": "value"} + with self.assertRaises(ApiError): + self.consumer.get_component(request) + + self.session._instance.get.assert_called_once_with( + url="http://fakehost/api/v2/components/", + headers={ + "Authorization": "Token fake_token", + "Content-Type": "application/json", + }, + params=request, + verify=False, + ) + mock_logger.error.assert_not_called() + + @patch( + "devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.component.VERIFY_CERTIFICATE", + True, + ) + def test_post_component_success(self): + mock_response = MagicMock() + mock_response.status_code = 201 + mock_response.json.return_value = {"id": 1, "name": "component"} + self.session._instance.post.return_value = mock_response + + request = {"name": "component"} + response = self.consumer.post_component(request) + + self.session._instance.post.assert_called_once_with( + url="http://fakehost/api/v2/components/", + headers={ + "Authorization": "Token fake_token", + "Content-Type": "application/json", + }, + data=json.dumps(request), + verify=True, + ) + self.assertEqual(response.name, "component") + + + def test_post_component_failure(self): + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.json.return_value = {"error": "Bad Request"} + self.session._instance.post.return_value = mock_response + + request = {"name": "component"} + with self.assertRaises(ApiError): + self.consumer.post_component(request) + + self.session._instance.post.assert_called_once_with( + url="http://fakehost/api/v2/components/", + headers={ + "Authorization": "Token fake_token", + "Content-Type": "application/json", + }, + data=json.dumps(request), + verify=False, + ) + + def test_post_component_exception(self): + self.session._instance.post.side_effect = Exception("Some error") + + request = {"name": "component"} + with self.assertRaises(ApiError): + self.consumer.post_component(request) + + self.session._instance.post.assert_called_once_with( + url="http://fakehost/api/v2/components/", + headers={ + "Authorization": "Token fake_token", + "Content-Type": "application/json", + }, + data=json.dumps(request), + verify=False, + ) + diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_finding_exclusion.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_finding_exclusion.py new file mode 100644 index 000000000..42b9d5170 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_finding_exclusion.py @@ -0,0 +1,57 @@ +import unittest +from unittest.mock import patch, MagicMock +from devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.finding_exclusion import FindingExclusionRestConsumer +from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError + +class TestFindingExclusionRestConsumer(unittest.TestCase): + + @patch('devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.finding_exclusion.SessionManager') + def setUp(self, MockSessionManager): + self.mock_session = MockSessionManager.return_value + self.mock_session._token = 'fake_token' + self.mock_session._host = 'http://fakehost' + self.mock_session._instance = MagicMock() + self.consumer = FindingExclusionRestConsumer(self.mock_session) + + @patch('devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.finding_exclusion.FindingExclusionList') + def test_get_finding_exclusions_success(self, MockFindingExclusionList): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {'key': 'value'} + self.mock_session._instance.get.return_value = mock_response + MockFindingExclusionList.from_dict.return_value = 'finding_exclusion_object' + + request = {'param': 'value'} + result = self.consumer.get_finding_exclusions(request) + + self.mock_session._instance.get.assert_called_once_with( + 'http://fakehost/api/v2/finding_exclusions/', + headers={'Authorization': 'Token fake_token', 'Content-Type': 'application/json'}, + params=request, + verify=False + ) + MockFindingExclusionList.from_dict.assert_called_once_with({'key': 'value'}) + self.assertEqual(result, 'finding_exclusion_object') + + def test_get_finding_exclusions_api_error(self): + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.json.return_value = {'error': 'some error'} + self.mock_session._instance.get.return_value = mock_response + + request = {'param': 'value'} + with self.assertRaises(ApiError): + self.consumer.get_finding_exclusions(request) + + @patch('devsecops_engine_tools.engine_utilities.defect_dojo.infraestructure.driver_adapters.finding_exclusion.logger') + def test_get_finding_exclusions_exception(self, mock_logger): + self.mock_session._instance.get.side_effect = Exception('some exception') + + request = {'param': 'value'} + with self.assertRaises(ApiError): + self.consumer.get_finding_exclusions(request) + + mock_logger.error.assert_called_once_with('from dict FindingExclusion: some exception') + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_product.py b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_product.py index 8eed484b6..486e7a4ec 100644 --- a/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_product.py +++ b/tools/devsecops_engine_tools/engine_utilities/defect_dojo/test/infraestucture/driver_adapter/test_product.py @@ -15,7 +15,7 @@ def test_get_product_info_success(): session_mock = session_manager_get(status_code=200, response_json_file="product_list.json") request = ImportScanRequest() - rest_product = ProductRestConsumer(request, session_mock) + rest_product = ProductRestConsumer(session_mock) product_obj = rest_product.get_products(request) # Verificar el resultado assert isinstance(product_obj, ProductList) @@ -29,14 +29,14 @@ def test_get_product_info_success(): def test_get_product_info_failure(): session_mock = session_manager_get(status_code=500, response_json_file="product_list.json") request = ImportScanRequest() - rest_product = ProductRestConsumer(ImportScanRequest(), session_mock) + rest_product = ProductRestConsumer(session_mock) with pytest.raises(ApiError): rest_product.get_products(request) def test_post_product_info_sucessfull(): session_mock = session_manager_post(status_code=201, mock_response="product.json") - rest_product = ProductRestConsumer(ImportScanRequest(), session_mock) + rest_product = ProductRestConsumer(session_mock) request = ImportScanRequest() request.product_name = "NU0212001_product name test_NU0212001" response = rest_product.post_product(request, 278) @@ -50,6 +50,6 @@ def test_post_product_info_sucessfull(): def test_post_product_info_failure(): session_mock = session_manager_post(status_code=500, mock_response="product.json") - rest_product_type = ProductRestConsumer(ImportScanRequest(), session_mock) + rest_product_type = ProductRestConsumer(session_mock) with pytest.raises(ApiError): rest_product_type.post_product(ImportScanRequest(), 278) diff --git a/tools/devsecops_engine_tools/engine_utilities/github/infrastructure/github_api.py b/tools/devsecops_engine_tools/engine_utilities/github/infrastructure/github_api.py old mode 100644 new mode 100755 index da5293e8b..e7b27d060 --- a/tools/devsecops_engine_tools/engine_utilities/github/infrastructure/github_api.py +++ b/tools/devsecops_engine_tools/engine_utilities/github/infrastructure/github_api.py @@ -1,28 +1,28 @@ import requests import zipfile import json -from github import Github +from github import Github, GithubIntegration from devsecops_engine_tools.engine_utilities.utils.api_error import ApiError class GithubApi: - def __init__( - self, - personal_access_token: str = "" - ): - self.__personal_access_token = personal_access_token def unzip_file(self, zip_file_path, extract_path): with zipfile.ZipFile(zip_file_path, "r") as zip_ref: zip_ref.extractall(extract_path) + + def get_installation_access_token(self,private_key,app_id,instalation_id): + if private_key: + private_key = private_key.replace("\\n", "\n") + integration = GithubIntegration(app_id, private_key) + access_token = integration.get_access_token(instalation_id) + return access_token.token def download_latest_release_assets( - self, owner, repository, download_path="." - ): + self, owner, repository, token, download_path=".", + ): url = f"https://api.github.com/repos/{owner}/{repository}/releases/latest" - - headers = {"Authorization": f"token {self.__personal_access_token}"} - + headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github+json"} response = requests.get(url, headers=headers) if response.status_code == 200: @@ -32,11 +32,8 @@ def download_latest_release_assets( for asset in assets: asset_url = asset["url"] asset_name = asset["name"] - headers.update({"Accept": "application/octet-stream"}) - response = requests.get(asset_url, headers=headers, stream=True) - if response.status_code == 200: with open(f"{download_path}/{asset_name}", "wb") as file: for chunk in response.iter_content(chunk_size=8192): @@ -51,18 +48,20 @@ def download_latest_release_assets( f"Error getting the assets of the last release. Status code: {response.status_code}" ) - def get_github_connection(self): - git_client = Github(self.__personal_access_token) - + def get_github_connection(self,personal_access_token): + git_client = Github(personal_access_token) return git_client - def get_remote_json_config(self, git_client: Github, owner, repository, path): + def get_remote_json_config(self, git_client: Github, owner, repository, path, branch=""): try: repo = git_client.get_repo(f"{owner}/{repository}") - file_content = repo.get_contents(path) + + if branch: file_content = repo.get_contents(path, ref=branch) + else: file_content = repo.get_contents(path) + data = file_content.decoded_content.decode() content_json = json.loads(data) return content_json except Exception as e: - raise ApiError("Error getting remote github configuration file: " + str(e)) + raise ApiError("Error getting remote github configuration file: " + str(e)) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/github/models/GithubPredefinedVariables.py b/tools/devsecops_engine_tools/engine_utilities/github/models/GithubPredefinedVariables.py index f74d991b3..720afa642 100644 --- a/tools/devsecops_engine_tools/engine_utilities/github/models/GithubPredefinedVariables.py +++ b/tools/devsecops_engine_tools/engine_utilities/github/models/GithubPredefinedVariables.py @@ -54,3 +54,9 @@ class AgentVariables(BaseEnum): github_workspace = "github.workspace" runner_os = "runner.os" runner_tool_cache = "runner.tool.cache" + + +class VMVariables(BaseEnum): + Vm_Product_Type_Name = "Vm.Product.Type.Name" + Vm_Product_Name = "Vm.Product.Name" + Vm_Product_Description = "Vm.Product.Description" diff --git a/tools/devsecops_engine_tools/engine_utilities/sbom/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sbom/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sbom/deserealizator.py b/tools/devsecops_engine_tools/engine_utilities/sbom/deserealizator.py new file mode 100644 index 000000000..2355c1013 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sbom/deserealizator.py @@ -0,0 +1,24 @@ +import json + +from devsecops_engine_tools.engine_core.src.domain.model.component import Component + + +def get_list_component(result_sbom, format) -> "list[Component]": + list_components = [] + + with open(result_sbom, "rb") as file: + sbom_object = file.read() + json_data = json.loads(sbom_object) + + if "cyclonedx" in format: + for component in json_data.get("components", []): + if component.get("version") != "UNKNOWN": + component_name = ( + f"{component.get('group','')}_{component.get('name')}" + if component.get("group") + else component.get("name") + ) + list_components.append( + Component(component_name, component.get("version")) + ) + return list_components diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/applications/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/applications/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/applications/runner_report_sonar.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/applications/runner_report_sonar.py new file mode 100644 index 000000000..bb6890ca0 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/applications/runner_report_sonar.py @@ -0,0 +1,119 @@ +from devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.aws.secrets_manager import ( + SecretsManager +) +from devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.azure.azure_devops import ( + AzureDevops +) +from devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.defect_dojo.defect_dojo import ( + DefectDojoPlatform +) +from devsecops_engine_tools.engine_utilities.sonarqube.src.infrastructure.driven_adapters.sonarqube.sonarqube_report import( + SonarAdapter +) +from devsecops_engine_tools.engine_core.src.infrastructure.driven_adapters.aws.s3_manager import ( + S3Manager, +) +from devsecops_engine_tools.engine_utilities.sonarqube.src.infrastructure.entry_points.entry_point_report_sonar import ( + init_report_sonar +) +import sys +import argparse +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities import settings + +logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() + +def get_inputs_from_cli(args): + parser = argparse.ArgumentParser() + parser.add_argument( + "-rcf", + "--remote_config_repo", + type=str, + required=True, + help="Name of Config Repo", + ) + parser.add_argument( + "-rcb", + "--remote_config_branch", + type=str, + required=False, + default="", + help="Name of the branch of Config Repo", + ) + parser.add_argument( + "--use_secrets_manager", + choices=["true", "false"], + type=str, + required=True, + help="Use Secrets Manager to get the tokens", + ) + parser.add_argument( + "--send_metrics", + choices=["true", "false"], + type=str, + required=False, + help="Enable or Disable the send metrics to the driven adapter metrics", + ) + parser.add_argument( + "--sonar_url", + required=False, + help="Url to access sonar API", + ) + parser.add_argument( + "--token_cmdb", + required=False, + help="Token to connect to the CMDB" + ) + parser.add_argument( + "--token_vulnerability_management", + required=False, + help="Token to connect to the Vulnerability Management", + ) + parser.add_argument( + "--token_sonar", + required=False, + help="Token to access sonar server", + ) + + args = parser.parse_args() + return { + "remote_config_repo": args.remote_config_repo, + "remote_config_branch": args.remote_config_branch, + "use_secrets_manager": args.use_secrets_manager, + "send_metrics": args.send_metrics, + "sonar_url": args.sonar_url, + "token_cmdb": args.token_cmdb, + "token_vulnerability_management": args.token_vulnerability_management, + "token_sonar": args.token_sonar, + } + +def runner_report_sonar(): + try: + vulnerability_management_gateway = DefectDojoPlatform() + secrets_manager_gateway = SecretsManager() + devops_platform_gateway = AzureDevops() + sonar_gateway = SonarAdapter() + metrics_manager_gateway = S3Manager() + args = get_inputs_from_cli(sys.argv[1:]) + + init_report_sonar( + vulnerability_management_gateway=vulnerability_management_gateway, + secrets_manager_gateway=secrets_manager_gateway, + devops_platform_gateway=devops_platform_gateway, + sonar_gateway=sonar_gateway, + metrics_manager_gateway=metrics_manager_gateway, + args=args, + ) + + except Exception as e: + logger.error("Error report_sonar: {0} ".format(str(e))) + print( + devops_platform_gateway.message( + "error", "Error report_sonar: {0} ".format(str(e)) + ) + ) + print(devops_platform_gateway.result_pipeline("failed")) + + +if __name__ == "__main__": + runner_report_sonar() \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/model/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/model/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/model/gateways/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/model/gateways/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/model/gateways/sonar_gateway.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/model/gateways/sonar_gateway.py new file mode 100644 index 000000000..2470b72fe --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/model/gateways/sonar_gateway.py @@ -0,0 +1,63 @@ +from abc import ( + ABCMeta, + abstractmethod +) + +class SonarGateway(metaclass=ABCMeta): + @abstractmethod + def get_project_keys( + self, + pipeline_name: str + ): + "get sonar project keys" + + @abstractmethod + def parse_project_key( + self, + file_path: str + ): + "find project key in metadata file" + + @abstractmethod + def create_task_report_from_string( + self, + file_content: str + ): + "make dict from metadata file" + + @abstractmethod + def filter_by_sonarqube_tag( + self, + findings: list + ): + "search for sonar findings" + + @abstractmethod + def change_finding_status( + self, + sonar_url: str, + sonar_token: str, + endpoint: str, + data: dict, + finding_type: str + ): + "use API to change vulnerabilities state in sonar" + + @abstractmethod + def get_findings( + self, + sonar_url: str, + sonar_token: str, + endpoint: str, + params: dict, + finding_type: str + ): + "use API to get project findings in sonar" + + @abstractmethod + def search_finding_by_id( + self, + findings: list, + finding_id: str + ): + "search a finding by id" \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/usecases/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/usecases/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/usecases/report_sonar.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/usecases/report_sonar.py new file mode 100644 index 000000000..5ea045ed5 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/domain/usecases/report_sonar.py @@ -0,0 +1,201 @@ +from devsecops_engine_tools.engine_utilities.sonarqube.src.infrastructure.helpers.utils import ( + set_repository +) +from devsecops_engine_tools.engine_core.src.infrastructure.helpers.util import ( + define_env +) +from devsecops_engine_tools.engine_core.src.domain.model.gateway.vulnerability_management_gateway import ( + VulnerabilityManagementGateway +) +from devsecops_engine_tools.engine_core.src.domain.model.vulnerability_management import ( + VulnerabilityManagement +) +from devsecops_engine_tools.engine_core.src.domain.model.gateway.secrets_manager_gateway import ( + SecretsManagerGateway +) +from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import ( + DevopsPlatformGateway +) +from devsecops_engine_tools.engine_utilities.sonarqube.src.domain.model.gateways.sonar_gateway import ( + SonarGateway +) +from devsecops_engine_tools.engine_core.src.domain.model.input_core import ( + InputCore +) +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities import settings + +logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() + +class ReportSonar: + def __init__( + self, + vulnerability_management_gateway: VulnerabilityManagementGateway, + secrets_manager_gateway: SecretsManagerGateway, + devops_platform_gateway: DevopsPlatformGateway, + sonar_gateway: SonarGateway + ): + self.vulnerability_management_gateway = vulnerability_management_gateway + self.secrets_manager_gateway = secrets_manager_gateway + self.devops_platform_gateway = devops_platform_gateway + self.sonar_gateway = sonar_gateway + + def process(self, args): + pipeline_name = self.devops_platform_gateway.get_variable("pipeline_name") + branch = self.devops_platform_gateway.get_variable("branch_tag").replace("refs/heads/", "") + input_core = InputCore( + [], + {}, + "", + "", + "", + self.devops_platform_gateway.get_variable("stage").capitalize(), + ) + + compact_remote_config_url = self.devops_platform_gateway.get_base_compact_remote_config_url(args["remote_config_repo"]) + source_code_management_uri = set_repository( + pipeline_name, + self.devops_platform_gateway.get_source_code_management_uri() + ) + config_tool = self.devops_platform_gateway.get_remote_config( + args["remote_config_repo"], + "/engine_core/ConfigTool.json", + args["remote_config_branch"] + ) + environment = define_env(None, branch) + + if args["use_secrets_manager"] == "true": + secret = self.secrets_manager_gateway.get_secret(config_tool) + secret_tool = secret + else: + secret = args + secret_tool = None + + report_config_tool = self.devops_platform_gateway.get_remote_config( + args["remote_config_repo"], + "/report_sonar/ConfigTool.json", + args["remote_config_branch"] + ) + + get_components = report_config_tool["PIPELINE_COMPONENTS"].get(pipeline_name) + if get_components: + project_keys = [f"{pipeline_name}_{component}" for component in get_components] + print(f"Multiple project keys detected: {project_keys}") + logger.info(f"Multiple project keys detected: {project_keys}") + else: + project_keys = self.sonar_gateway.get_project_keys(pipeline_name) + + args["tool"] = "sonarqube" + vulnerability_manager = VulnerabilityManagement( + scan_type = "SONARQUBE", + input_core = input_core, + dict_args = args, + secret_tool = secret_tool, + config_tool = config_tool, + source_code_management_uri = source_code_management_uri, + base_compact_remote_config_url = compact_remote_config_url, + access_token = self.devops_platform_gateway.get_variable("access_token"), + version = self.devops_platform_gateway.get_variable("build_execution_id"), + build_id = self.devops_platform_gateway.get_variable("build_id"), + branch_tag = branch, + commit_hash = self.devops_platform_gateway.get_variable("commit_hash"), + environment = environment, + vm_product_type_name = self.devops_platform_gateway.get_variable("vm_product_type_name"), + vm_product_name = self.devops_platform_gateway.get_variable("vm_product_name"), + vm_product_description = self.devops_platform_gateway.get_variable("vm_product_description"), + ) + + for project_key in project_keys: + try: + findings = self.vulnerability_management_gateway.get_all( + service=project_key, + dict_args=args, + secret_tool=secret_tool, + config_tool=config_tool + )[0] + filtered_findings = self.sonar_gateway.filter_by_sonarqube_tag(findings) + + sonar_vulnerabilities = self.sonar_gateway.get_findings( + args["sonar_url"], + secret["token_sonar"], + "/api/issues/search", + { + "componentKeys": project_key, + "types": "VULNERABILITY", + "ps": 500, + "p": 1, + "s": "CREATION_DATE", + "asc": "false" + }, + "issues" + ) + sonar_hotspots = self.sonar_gateway.get_findings( + args["sonar_url"], + secret["token_sonar"], + "/api/hotspots/search", + { + "projectKey": project_key, + "ps": 100, + "p": 1, + }, + "hotspots" + ) + + sonar_findings = sonar_vulnerabilities + sonar_hotspots + + for finding in filtered_findings: + related_sonar_finding = self.sonar_gateway.search_finding_by_id( + sonar_findings, + finding.unique_id_from_tool + ) + status = None + if related_sonar_finding: + if related_sonar_finding.get("type") == "VULNERABILITY": + if finding.active and related_sonar_finding["status"] == "RESOLVED": status = "reopen" + elif related_sonar_finding["status"] != "RESOLVED": + if finding.false_p: status = "falsepositive" + elif finding.risk_accepted or finding.out_of_scope: status = "wontfix" + if status: + self.sonar_gateway.change_finding_status( + args["sonar_url"], + secret["token_sonar"], + "/api/issues/do_transition", + { + "issue": related_sonar_finding["key"], + "transition": status + }, + "issue" + ) + else: + resolution = None + if finding.active and related_sonar_finding["status"] == "REVIEWED": status = "TO_REVIEW" + elif related_sonar_finding["status"] == "TO_REVIEW": + if finding.false_p: resolution = "SAFE" + elif finding.risk_accepted or finding.out_of_scope: resolution = "ACKNOWLEDGED" + if resolution: status = "REVIEWED" + if status: + data = { + "hotspot": related_sonar_finding["key"], + "status": status, + "resolution": resolution + } + if not resolution: data.pop("resolution") + self.sonar_gateway.change_finding_status( + args["sonar_url"], + secret["token_sonar"], + "/api/hotspots/change_status", + data, + "hotspot" + ) + + except Exception as e: + logger.warning(f"It was not possible to synchronize Sonar and Vulnerability Manager: {e}") + + input_core.scope_pipeline = project_key + + self.vulnerability_management_gateway.send_vulnerability_management( + vulnerability_management=vulnerability_manager + ) + + input_core.scope_pipeline = pipeline_name + return input_core \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/driven_adapters/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/driven_adapters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/driven_adapters/sonarqube/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/driven_adapters/sonarqube/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/driven_adapters/sonarqube/sonarqube_report.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/driven_adapters/sonarqube/sonarqube_report.py new file mode 100644 index 000000000..1ccea5872 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/driven_adapters/sonarqube/sonarqube_report.py @@ -0,0 +1,112 @@ +from devsecops_engine_tools.engine_utilities.utils.utils import ( + Utils +) +from devsecops_engine_tools.engine_utilities.sonarqube.src.domain.model.gateways.sonar_gateway import ( + SonarGateway +) +import os +import re +import requests +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities import settings + +logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() + +class SonarAdapter(SonarGateway): + def get_project_keys(self, pipeline_name): + project_keys = [pipeline_name] + sonar_scanner_params = os.getenv("SONARQUBE_SCANNER_PARAMS", "") + pattern = r'"sonar\.scanner\.metadataFilePath":"(.*?)"' + match_result = re.search(pattern, sonar_scanner_params) + + if match_result and match_result.group(1): + metadata_file_path = match_result.group(1) + project_key_found = self.parse_project_key(metadata_file_path) + + if project_key_found: + print(f"ProjectKey scanner params: {project_key_found}") + project_keys = [project_key_found] + + return project_keys + + def parse_project_key(self, file_path): + try: + with open(file_path, 'r', encoding='utf-8') as f: + file_content = f.read() + print(f"[SQ] Parse Task report file:\n{file_content}") + if not file_content or len(file_content) <= 0: + print("[SQ] Error reading file") + logger.warning("[SQ] Error reading file") + return None + try: + settings = self.create_task_report_from_string(file_content) + return settings.get("projectKey") + except Exception as err: + print(f"[SQ] Parse Task report error: {err}") + logger.warning(f"[SQ] Parse Task report error: {err}") + return None + except Exception as err: + logger.warning(f"[SQ] Error reading file: {str(err)}") + return None + + def create_task_report_from_string(self, file_content): + lines = file_content.replace('\r\n', '\n').split('\n') + settings = {} + for line in lines: + split_line = line.split('=') + if len(split_line) > 1: + settings[split_line[0]] = '='.join(split_line[1:]) + return settings + + def filter_by_sonarqube_tag(self, findings): + return [finding for finding in findings if "sonarqube" in finding.tags] + + def change_finding_status(self, sonar_url, sonar_token, endpoint, data, finding_type): + try: + response = requests.post( + f"{sonar_url}{endpoint}", + headers={ + "Authorization": f"Basic {Utils().encode_token_to_base64(sonar_token)}" + }, + data=data + ) + response.raise_for_status() + + if finding_type == "issue": + info = data["transition"] + else: + resolution_info = "" + if data.get("resolution"): resolution_info = f" ({data['resolution']})" + + info = f"{data['status']}{resolution_info}" + + print(f"The state of the {finding_type} {data[finding_type]} was changed to {info}.") + except Exception as e: + logger.warning(f"Unable to change the status of {finding_type} {data[finding_type]}. Error: {e}") + pass + + def get_findings(self, sonar_url, sonar_token, endpoint, params, finding_type): + findings = [] + try: + while True: + response = requests.get( + f"{sonar_url}{endpoint}", + headers={ + "Authorization": f"Basic {Utils().encode_token_to_base64(sonar_token)}" + }, + params=params + ) + response.raise_for_status() + data = response.json() + + findings.extend(data[finding_type]) + if len(data[finding_type]) < params["ps"]: break + params["p"] = params["p"] + 1 + + return findings + except Exception as e: + logger.warning(f"It was not possible to obtain the {finding_type}: {str(e)}") + return [] + + def search_finding_by_id(self, issues, issue_id): + return next((issue for issue in issues if issue["key"] in issue_id), None) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/entry_points/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/entry_points/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/entry_points/entry_point_report_sonar.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/entry_points/entry_point_report_sonar.py new file mode 100644 index 000000000..99e43bdca --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/entry_points/entry_point_report_sonar.py @@ -0,0 +1,66 @@ +from devsecops_engine_tools.engine_utilities.sonarqube.src.domain.usecases.report_sonar import ( + ReportSonar, +) +from devsecops_engine_tools.engine_utilities.utils.printers import ( + Printers, +) +from devsecops_engine_tools.engine_core.src.domain.usecases.metrics_manager import ( + MetricsManager, +) +import re +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities import settings + +logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() + + +def init_report_sonar( + vulnerability_management_gateway, + secrets_manager_gateway, + devops_platform_gateway, + sonar_gateway, + metrics_manager_gateway, + args, +): + config_tool = devops_platform_gateway.get_remote_config( + args["remote_config_repo"], "/engine_core/ConfigTool.json", args["remote_config_branch"] + ) + report_config_tool = devops_platform_gateway.get_remote_config( + args["remote_config_repo"], "/report_sonar/ConfigTool.json" + ) + Printers.print_logo_tool(config_tool["BANNER"]) + + pipeline_name = devops_platform_gateway.get_variable("pipeline_name") + branch = devops_platform_gateway.get_variable("branch_tag") + is_valid_pipeline = not re.match( + report_config_tool["IGNORE_SEARCH_PATTERN"], pipeline_name, re.IGNORECASE + ) + is_valid_branch = any( + target_branch in str(branch) + for target_branch in report_config_tool["TARGET_BRANCHES"] + ) + is_enabled = config_tool["REPORT_SONAR"]["ENABLED"] + + if is_enabled and is_valid_pipeline and is_valid_branch: + input_core = ReportSonar( + vulnerability_management_gateway, + secrets_manager_gateway, + devops_platform_gateway, + sonar_gateway, + ).process(args) + + if args["send_metrics"] == "true": + MetricsManager(devops_platform_gateway, metrics_manager_gateway).process( + config_tool, input_core, {"tool": "report_sonar"}, "" + ) + else: + if not is_enabled: + message = "DevSecOps Engine Tool - {0} in maintenance...".format( + "report_sonar" + ) + else: + message = "Tool skipped by DevSecOps policy" + + print( + devops_platform_gateway.message("warning", message), + ) diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/helpers/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/helpers/utils.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/helpers/utils.py new file mode 100644 index 000000000..796189ff2 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/src/infrastructure/helpers/utils.py @@ -0,0 +1,8 @@ +import re + +def set_repository(pipeline_name, source_code_management): + if re.search('_MR_', pipeline_name) is None: + return source_code_management + else: + splittedPipeline = pipeline_name.split('_MR_') + return source_code_management + '?path=/' + splittedPipeline[1] diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/applications/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/applications/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/applications/test_runner_report_sonar.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/applications/test_runner_report_sonar.py new file mode 100644 index 000000000..c0cf151f8 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/applications/test_runner_report_sonar.py @@ -0,0 +1,68 @@ +import unittest +from unittest.mock import patch +import sys +import argparse +from devsecops_engine_tools.engine_utilities.sonarqube.src.applications.runner_report_sonar import runner_report_sonar, get_inputs_from_cli + +class TestRunnerReportSonar(unittest.TestCase): + @patch( + "devsecops_engine_tools.engine_utilities.sonarqube.src.applications.runner_report_sonar.get_inputs_from_cli" + ) + @patch( + "devsecops_engine_tools.engine_utilities.sonarqube.src.applications.runner_report_sonar.init_report_sonar" + ) + def test_runner_report_sonar_success(self, mock_init_report_sonar, mock_get_inputs_from_cli): + # Act + runner_report_sonar() + + # Assert + mock_init_report_sonar.assert_called_once() + + @patch( + "devsecops_engine_tools.engine_utilities.sonarqube.src.applications.runner_report_sonar.get_inputs_from_cli" + ) + @patch( + "devsecops_engine_tools.engine_utilities.sonarqube.src.applications.runner_report_sonar.logger" + ) + def test_runner_report_sonar_exception(self, mock_logger, mock_get_inputs_from_cli): + # Arrange + mock_get_inputs_from_cli.side_effect = Exception("Test exception") + + # Act + runner_report_sonar() + + # Assert + mock_logger.error.assert_called_with("Error report_sonar: Test exception ") + + @patch( + "argparse.ArgumentParser.parse_args" + ) + def test_get_inputs_from_cli(self, mock_parse_args): + # Arrange + mock_parse_args.return_value = argparse.Namespace( + remote_config_repo="test_repo", + use_secrets_manager="false", + send_metrics="true", + sonar_url="https://sonar.com/", + token_cmdb="my_token_cmdb", + token_vulnerability_management="my_token_vm", + token_sonar="my_token_sonar", + remote_config_branch="" + ) + + expected_output = { + "remote_config_repo": "test_repo", + "use_secrets_manager": "false", + "send_metrics": "true", + "sonar_url": "https://sonar.com/", + "token_cmdb": "my_token_cmdb", + "token_vulnerability_management": "my_token_vm", + "token_sonar": "my_token_sonar", + "remote_config_branch": "" + } + + # Act + result = get_inputs_from_cli(sys.argv[1:]) + + # Assert + self.assertEqual(result, expected_output) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/domain/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/domain/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/domain/usecases/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/domain/usecases/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/domain/usecases/test_report_sonar.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/domain/usecases/test_report_sonar.py new file mode 100644 index 000000000..f17e6388a --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/domain/usecases/test_report_sonar.py @@ -0,0 +1,110 @@ +import unittest +from unittest.mock import MagicMock, patch, call +from devsecops_engine_tools.engine_utilities.sonarqube.src.domain.usecases.report_sonar import ( + ReportSonar +) + +class TestReportSonar(unittest.TestCase): + @patch( + "devsecops_engine_tools.engine_utilities.sonarqube.src.domain.usecases.report_sonar.set_repository" + ) + @patch( + "devsecops_engine_tools.engine_utilities.sonarqube.src.domain.usecases.report_sonar.define_env" + ) + def test_process_valid( + self, mock_define_env, mock_set_repository + ): + # Arrange + mock_vulnerability_gateway = MagicMock() + mock_secrets_manager_gateway = MagicMock() + mock_devops_platform_gateway = MagicMock() + mock_sonar_gateway = MagicMock() + + mock_devops_platform_gateway.get_variable.side_effect = [ + "pipeline_name", + "branch_name", + "repository", + "access_token", + "build_execution_id", + "build_id", + "commit_hash", + "repository_provider", + "vm_product_type_name", + "vm_product_name" + ] + mock_set_repository.return_value = "repository_uri" + mock_define_env.return_value = "dev" + mock_secrets_manager_gateway.get_secret.return_value = { + "token_sonar": "sonar_token" + } + + mock_devops_platform_gateway.get_remote_config.return_value = { + "PIPELINE_COMPONENTS": {} + } + + mock_sonar_gateway.get_project_keys.return_value = ["project_key_1"] + mock_sonar_gateway.filter_by_sonarqube_tag.return_value = [ + MagicMock(unique_id_from_tool="123", active=True, mitigated=False, false_p=False), + MagicMock(unique_id_from_tool="1234", active=False, mitigated=False, false_p=True) + ] + mock_sonar_gateway.search_finding_by_id.side_effect = [ + {"status": "RESOLVED", "key": "123", "type": "VULNERABILITY"}, + {"status": "REVIEWED", "key": "1234"} + ] + + report_sonar = ReportSonar( + vulnerability_management_gateway=mock_vulnerability_gateway, + secrets_manager_gateway=mock_secrets_manager_gateway, + devops_platform_gateway=mock_devops_platform_gateway, + sonar_gateway=mock_sonar_gateway, + ) + + args = {"remote_config_repo": "repo", "use_secrets_manager": "true", "sonar_url": "sonar_url", "remote_config_branch": ""} + + # Act + report_sonar.process(args) + + # Assert + mock_sonar_gateway.get_findings.assert_has_calls( + [ + call("sonar_url", + "sonar_token", + "/api/issues/search", + { + "componentKeys": "project_key_1", + "types": "VULNERABILITY", + "ps": 500, + "p": 1, + "s": "CREATION_DATE", + "asc": "false" + }, + "issues" + ), + call("sonar_url", + "sonar_token", + "/api/hotspots/search", + { + "projectKey": "project_key_1", + "ps": 100, + "p": 1 + }, + "hotspots" + ) + ], + any_order=False + ) + mock_sonar_gateway.change_finding_status.assert_has_calls( + [ + call( + "sonar_url", + "sonar_token", + "/api/issues/do_transition", + { + "issue": "123", + "transition": "reopen" + }, + "issue" + ) + ], + any_order=False + ) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/entry_points/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/entry_points/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/entry_points/test_entry_point_report_sonar.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/entry_points/test_entry_point_report_sonar.py new file mode 100644 index 000000000..ded1061fd --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/entry_points/test_entry_point_report_sonar.py @@ -0,0 +1,91 @@ +import unittest +from unittest.mock import MagicMock, patch +from devsecops_engine_tools.engine_utilities.sonarqube.src.infrastructure.entry_points.entry_point_report_sonar import init_report_sonar + +class TestInitReportSonar(unittest.TestCase): + + @patch( + "devsecops_engine_tools.engine_utilities.sonarqube.src.infrastructure.entry_points.entry_point_report_sonar.ReportSonar" + ) + def test_init_report_sonar_calls_process(self, mock_report_sonar): + # Arrange + mock_vulnerability_management_gateway = MagicMock() + mock_secrets_manager_gateway = MagicMock() + mock_devops_platform_gateway = MagicMock() + mock_metrics_manager_gateway = MagicMock() + mock_sonar_gateway = MagicMock() + mock_devops_platform_gateway.get_remote_config.side_effect = [ + { + "REPORT_SONAR" : { + "ENABLED": True + }, + "BANNER": "DevSecOps" + }, + { + "IGNORE_SEARCH_PATTERN": ".*test.*", + "TARGET_BRANCHES": ["trunk", "develop", "master"], + "PIPELINE_COMPONENTS": { + "EXAMPLE_MULTICOMPONENT_PIPELINE": [] + } + } + ] + + args = {"remote_config_repo": "some_repo", "use_secrets_manager": "true", "send_metrics": "false", "remote_config_branch": ""} + mock_devops_platform_gateway.get_variable.side_effect = ["pipeline_name", "trunk"] + + # Act + init_report_sonar( + vulnerability_management_gateway=mock_vulnerability_management_gateway, + secrets_manager_gateway=mock_secrets_manager_gateway, + devops_platform_gateway=mock_devops_platform_gateway, + sonar_gateway=mock_sonar_gateway, + metrics_manager_gateway=mock_metrics_manager_gateway, + args=args, + ) + + # Assert + mock_report_sonar.assert_called_once_with( + mock_vulnerability_management_gateway, + mock_secrets_manager_gateway, + mock_devops_platform_gateway, + mock_sonar_gateway + ) + mock_report_sonar.return_value.process.assert_called_once_with(args) + + @patch( + "devsecops_engine_tools.engine_utilities.sonarqube.src.infrastructure.entry_points.entry_point_report_sonar.ReportSonar" + ) + def test_init_report_sonar_disabled(self, mock_report_sonar): + # Arrange + mock_devops_platform_gateway = MagicMock() + mock_metrics_manager_gateway = MagicMock() + mock_devops_platform_gateway.get_remote_config.side_effect = [ + { + "REPORT_SONAR" : { + "ENABLED": False + }, + "BANNER": "DevSecOps" + }, + { + "IGNORE_SEARCH_PATTERN": ".*test.*", + "TARGET_BRANCHES": ["trunk", "develop", "master"], + "PIPELINE_COMPONENTS": { + "EXAMPLE_MULTICOMPONENT_PIPELINE": [] + } + } + ] + args = {"remote_config_repo": "some_repo", "use_secrets_manager": "true", "send_metrics": "false", "remote_config_branch": ""} + mock_devops_platform_gateway.get_variable.side_effect = ["pipeline_name", "develop"] + + # Act + init_report_sonar( + vulnerability_management_gateway=MagicMock(), + secrets_manager_gateway=MagicMock(), + devops_platform_gateway=mock_devops_platform_gateway, + sonar_gateway=MagicMock(), + metrics_manager_gateway=mock_metrics_manager_gateway, + args=args, + ) + + # Assert + mock_report_sonar.assert_not_called() \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/helpers/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/helpers/test_utils.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/helpers/test_utils.py new file mode 100644 index 000000000..b3be6b92c --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/helpers/test_utils.py @@ -0,0 +1,26 @@ +import unittest +from devsecops_engine_tools.engine_utilities.sonarqube.src.infrastructure.helpers.utils import set_repository + +class TestSonarUtils(unittest.TestCase): + + def test_set_repository_mr(self): + # Arrange + pipeline_name = "some_pipeline" + source_code_management = "https://example.com/repo" + + # Act + result = set_repository(pipeline_name, source_code_management) + + # Assert + self.assertEqual(result, source_code_management) + + def test_set_repository_not_mr(self): + # Arrange + pipeline_name = "some_pipeline_MR_123" + source_code_management = "https://example.com/repo" + + # Act + result = set_repository(pipeline_name, source_code_management) + + # Assert + self.assertEqual(result, "https://example.com/repo?path=/123") \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/sonar/__init__.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/sonar/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/sonar/test_report_sonar.py b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/sonar/test_report_sonar.py new file mode 100644 index 000000000..e26ec8ab0 --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/sonarqube/test/infrastructure/sonar/test_report_sonar.py @@ -0,0 +1,219 @@ +import unittest +from unittest.mock import patch, mock_open, MagicMock +from devsecops_engine_tools.engine_utilities.sonarqube.src.infrastructure.driven_adapters.sonarqube.sonarqube_report import SonarAdapter + +class TestSonarAdapter(unittest.TestCase): + + @patch( + "os.getenv" + ) + def test_get_project_keys_from_env(self, mock_getenv): + # Arrange + adapter = SonarAdapter() + mock_getenv.return_value = '{"sonar.scanner.metadataFilePath":"path/to/metadata.json"}' + + with patch.object(adapter, 'parse_project_key', return_value="project_key_123") as mock_parse: + # Act + project_keys = adapter.get_project_keys("pipeline_name") + + # Assert + mock_parse.assert_called_once_with("path/to/metadata.json") + self.assertEqual(project_keys, ["project_key_123"]) + + @patch( + "os.getenv" + ) + def test_get_project_keys_no_match_in_env(self, mock_getenv): + # Arrange + adapter = SonarAdapter() + mock_getenv.return_value = "" + + # Act + project_keys = adapter.get_project_keys("pipeline_name") + + # Assert + self.assertEqual(project_keys, ["pipeline_name"]) + + @patch( + "os.getenv" + ) + def test_get_project_keys_no_project_key_found(self, mock_getenv): + # Arrange + adapter = SonarAdapter() + mock_getenv.return_value = '{"sonar.scanner.metadataFilePath":"path/to/metadata.json"}' + + with patch.object(adapter, "parse_project_key", return_value=None) as mock_parse: + # Act + project_keys = adapter.get_project_keys("pipeline_name") + + # Assert + mock_parse.assert_called_once_with("path/to/metadata.json") + self.assertEqual(project_keys, ["pipeline_name"]) + + @patch( + "builtins.open", + new_callable=mock_open, + read_data="projectKey=my_project_key" + ) + def test_parse_project_key_success(self, mock_file): + # Arrange + adapter = SonarAdapter() + + # Act + result = adapter.parse_project_key("path/to/metadata.json") + + # Assert + mock_file.assert_called_once_with("path/to/metadata.json", "r", encoding="utf-8") + self.assertEqual(result, "my_project_key") + + def test_parse_project_key_invalid_content(self): + # Arrange + adapter = SonarAdapter() + + # Act + result = adapter.parse_project_key("path/to/metadata.json") + + # Assert + self.assertIsNone(result) + + @patch( + "builtins.open", + side_effect=Exception("File not found") + ) + def test_parse_project_key_file_not_found(self, mock_file): + # Arrange + adapter = SonarAdapter() + + # Act + result = adapter.parse_project_key("path/to/nonexistent_file.json") + + # Assert + mock_file.assert_called_once_with("path/to/nonexistent_file.json", "r", encoding="utf-8") + self.assertIsNone(result) + + def test_create_task_report_from_string(self): + # Arrange + adapter = SonarAdapter() + file_content = "projectKey=my_project_key\nanotherSetting=some_value" + + # Act + result = adapter.create_task_report_from_string(file_content) + + # Assert + self.assertEqual(result["projectKey"], "my_project_key") + self.assertEqual(result["anotherSetting"], "some_value") + + @patch( + "requests.post" + ) + @patch( + "devsecops_engine_tools.engine_utilities.sonarqube.src.infrastructure.driven_adapters.sonarqube.sonarqube_report.Utils.encode_token_to_base64" + ) + def test_change_finding_status(self, mock_encode, mock_post): + # Arrange + mock_encode.return_value = "encoded_token" + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_post.return_value = mock_response + + sonar_adapter = SonarAdapter() + sonar_url = "https://sonar.example.com" + sonar_token = "my_token" + endpoint = "/api/issues/do_transition" + data = { + "issue": "123", + "transition": "reopen" + } + + # Act + sonar_adapter.change_finding_status( + sonar_url, + sonar_token, + endpoint, + data, + "issue" + ) + + # Assert + mock_post.assert_called_once_with( + f"{sonar_url}{endpoint}", + headers={"Authorization": "Basic encoded_token"}, + data={"issue": "123", "transition": "reopen"} + ) + mock_response.raise_for_status.assert_called_once() + + @patch( + "requests.get" + ) + @patch( + "devsecops_engine_tools.engine_utilities.sonarqube.src.infrastructure.driven_adapters.sonarqube.sonarqube_report.Utils.encode_token_to_base64" + ) + def test_get_findings(self, mock_encode, mock_get): + # Arrange + mock_encode.return_value = "encoded_token" + mock_response = MagicMock() + mock_response.json.return_value = { + "issues": [{"key": "123", "type": "VULNERABILITY"}] + } + mock_response.raise_for_status = MagicMock() + mock_get.return_value = mock_response + + report_sonar = SonarAdapter() + sonar_url = "https://sonar.example.com" + sonar_token = "my_token" + endpoint = "/api/issues/search" + params = { + "componentKeys": "my_project", + "types": "VULNERABILITY", + "ps": 500, + "p": 1, + "s": "CREATION_DATE", + "asc": "false" + } + + # Act + findings = report_sonar.get_findings( + sonar_url, + sonar_token, + endpoint, + params, + "issues" + ) + + # Assert + mock_get.assert_called_once_with( + f"{sonar_url}{endpoint}", + headers={"Authorization": "Basic encoded_token"}, + params=params + ) + mock_response.raise_for_status.assert_called_once() + self.assertEqual(findings, [{"key": "123", "type": "VULNERABILITY"}]) + + def test_search_finding_by_id(self): + # Arrange + report_sonar = SonarAdapter() + issues = [ + {"key": "123", "type": "VULNERABILITY"}, + {"key": "456", "type": "BUG"} + ] + issue_id = "123" + + # Act + result = report_sonar.search_finding_by_id(issues, issue_id) + + # Assert + self.assertEqual(result, {"key": "123", "type": "VULNERABILITY"}) + + def test_search_finding_by_id_not_found(self): + # Arrange + report_sonar = SonarAdapter() + issues = [ + {"key": "456", "type": "BUG"} + ] + issue_id = "999" + + # Act + result = report_sonar.search_finding_by_id(issues, issue_id) + + # Assert + self.assertIsNone(result) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_utilities/test/azuredevops/models/test_AzurePredefinedVariables.py b/tools/devsecops_engine_tools/engine_utilities/test/azuredevops/models/test_AzurePredefinedVariables.py index 251e401b3..18b466057 100644 --- a/tools/devsecops_engine_tools/engine_utilities/test/azuredevops/models/test_AzurePredefinedVariables.py +++ b/tools/devsecops_engine_tools/engine_utilities/test/azuredevops/models/test_AzurePredefinedVariables.py @@ -5,6 +5,7 @@ BuildVariables, ReleaseVariables, AgentVariables, + VMVariables ) @@ -24,6 +25,9 @@ (ReleaseVariables.Artifact_Path, "ARTIFACT_PATH", "ArtifactPathValue"), (AgentVariables.Agent_WorkFolder, "AGENT_WORKFOLDER", "AgentWorkFolder"), (AgentVariables.Agent_BuildDirectory, "AGENT_BUILDDIRECTORY", "AgentBuildDirectory"), + (VMVariables.Vm_Product_Type_Name, "VM_PRODUCT_TYPE_NAME", "ProductTypeName"), + (VMVariables.Vm_Product_Name, "VM_PRODUCT_NAME", "ProductName"), + (VMVariables.Vm_Product_Description, "VM_PRODUCT_DESCRIPTION", "ProductDescription"), ], ) def test_enum_env_name(monkeypatch, enum_class, expected_env_name, expected_value): diff --git a/tools/devsecops_engine_tools/engine_utilities/test/github/infrastructure/test_github_api.py b/tools/devsecops_engine_tools/engine_utilities/test/github/infrastructure/test_github_api.py old mode 100644 new mode 100755 index 3414b4fe6..757ccc930 --- a/tools/devsecops_engine_tools/engine_utilities/test/github/infrastructure/test_github_api.py +++ b/tools/devsecops_engine_tools/engine_utilities/test/github/infrastructure/test_github_api.py @@ -7,7 +7,7 @@ class TestGithubApi(unittest.TestCase): def setUp(self): self.personal_access_token = "your_token" - self.github_api = GithubApi(personal_access_token=self.personal_access_token) + self.github_api = GithubApi() @patch('devsecops_engine_tools.engine_utilities.github.infrastructure.github_api.zipfile.ZipFile') def test_unzip_file(self, mock_zipfile): @@ -61,9 +61,9 @@ def test_get_github_connection(self, mock_github): test_token = "test_token" - github_api = GithubApi(test_token) + github_api = GithubApi() - result = github_api.get_github_connection() + result = github_api.get_github_connection(test_token) mock_github.assert_called_once_with(test_token) @@ -86,7 +86,7 @@ def test_get_remote_json_config(self, MockGithub): mock_repo.get_contents.return_value = mock_file_content MockGithub.return_value = mock_github_instance - github_api = GithubApi("test_token") + github_api = GithubApi() result = github_api.get_remote_json_config(mock_github_instance, owner, repository, path) @@ -100,14 +100,10 @@ def test_get_remote_json_config_raises_error(self, MockGithub): owner = "test_owner" repository = "test_repo" path = "path/to/config.json" - mock_github_instance = MagicMock() mock_github_instance.get_repo.side_effect = Exception("Test exception") - MockGithub.return_value = mock_github_instance - - github_api = GithubApi("test_token") - + github_api = GithubApi() with self.assertRaises(ApiError) as context: github_api.get_remote_json_config(mock_github_instance, owner, repository, path) diff --git a/tools/devsecops_engine_tools/engine_utilities/test/sbom/__init__.py b/tools/devsecops_engine_tools/engine_utilities/test/sbom/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/devsecops_engine_tools/engine_utilities/test/sbom/test_deserealizator.py b/tools/devsecops_engine_tools/engine_utilities/test/sbom/test_deserealizator.py new file mode 100644 index 000000000..97708835d --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/test/sbom/test_deserealizator.py @@ -0,0 +1,72 @@ +import unittest +import json +from unittest.mock import mock_open, patch +from devsecops_engine_tools.engine_utilities.sbom.deserealizator import ( + get_list_component, +) +from devsecops_engine_tools.engine_core.src.domain.model.component import Component + + +class TestGetListComponent(unittest.TestCase): + @patch( + "builtins.open", + new_callable=mock_open, + read_data=json.dumps( + { + "components": [ + {"group": "group1", "name": "component1", "version": "1.0.0"}, + {"group": "group2", "name": "component2", "version": "2.0.0"}, + {"group": "group3", "name": "component3", "version": "UNKNOWN"}, + ] + } + ), + ) + def test_get_list_component_cyclonedx(self, mock_file): + result_sbom = "dummy_path" + format = "cyclonedx" + expected_components = [ + Component("group1_component1", "1.0.0"), + Component("group2_component2", "2.0.0"), + ] + + components = get_list_component(result_sbom, format) + + self.assertEqual(components, expected_components) + mock_file.assert_called_once_with(result_sbom, "rb") + + @patch( + "builtins.open", + new_callable=mock_open, + read_data=json.dumps({"components": []}), + ) + def test_get_list_component_empty(self, mock_file): + result_sbom = "dummy_path" + format = "cyclonedx" + expected_components = [] + + components = get_list_component(result_sbom, format) + + self.assertEqual(components, expected_components) + mock_file.assert_called_once_with(result_sbom, "rb") + + @patch( + "builtins.open", + new_callable=mock_open, + read_data=json.dumps( + { + "components": [ + {"name": "component1", "version": "1.0.0"}, + {"name": "component2", "version": "2.0.0"}, + ] + } + ), + ) + def test_get_list_component_non_cyclonedx_format(self, mock_file): + result_sbom = "dummy_path" + format = "other_format" + expected_components = [] + + components = get_list_component(result_sbom, format) + + self.assertEqual(components, expected_components) + mock_file.assert_called_once_with(result_sbom, "rb") diff --git a/tools/devsecops_engine_tools/engine_utilities/test/utils/test_utils.py b/tools/devsecops_engine_tools/engine_utilities/test/utils/test_utils.py new file mode 100644 index 000000000..e2d0c9e1a --- /dev/null +++ b/tools/devsecops_engine_tools/engine_utilities/test/utils/test_utils.py @@ -0,0 +1,82 @@ +from devsecops_engine_tools.engine_utilities.utils.utils import Utils + +def test_configurate_external_checks_git(): + json_data = { + "SEARCH_PATTERN": ["AW", "NU"], + "IGNORE_SEARCH_PATTERN": ["test"], + "MESSAGE_INFO_ENGINE_IAC": "message test", + "EXCLUSIONS_PATH": "Exclusions.json", + "UPDATE_SERVICE_WITH_FILE_NAME_CFT": "false", + "THRESHOLD": { + "VULNERABILITY": { + "Critical": 10, + "High": 3, + "Medium": 20, + "Low": 30, + }, + "COMPLIANCE": {"Critical": 4}, + }, + "CHECKOV": { + "VERSION": "2.3.296", + "USE_EXTERNAL_CHECKS_GIT": "True", + "EXTERNAL_CHECKS_GIT": "rules", + "EXTERNAL_GIT_SSH_HOST": "github", + "EXTERNAL_GIT_PUBLIC_KEY_FINGERPRINT": "fingerprint", + "USE_EXTERNAL_CHECKS_DIR": "False", + "EXTERNAL_DIR_OWNER": "test", + "EXTERNAL_DIR_REPOSITORY": "repository", + "EXTERNAL_DIR_ASSET_NAME": "rules", + "RULES": "", + "APP_ID_GITHUB": "app_id", + "INSTALATION_ID_GITHUB": "installation_id" + }, + } + + + util = Utils() + result = util.configurate_external_checks( + "checkov",json_data, None, "github_token:12234234" + ) + + assert result is None + + +def test_configurate_external_checks_dir(): + json_data = { + "SEARCH_PATTERN": ["AW", "NU"], + "IGNORE_SEARCH_PATTERN": [ + "test", + ], + "MESSAGE_INFO_ENGINE_IAC": "message test", + "EXCLUSIONS_PATH": "Exclusions.json", + "UPDATE_SERVICE_WITH_FILE_NAME_CFT": "false", + "THRESHOLD": { + "VULNERABILITY": { + "Critical": 10, + "High": 3, + "Medium": 20, + "Low": 30, + }, + "COMPLIANCE": {"Critical": 4}, + }, + "CHECKOV": { + "VERSION": "2.3.296", + "USE_EXTERNAL_CHECKS_GIT": "False", + "EXTERNAL_CHECKS_GIT": "rules", + "EXTERNAL_GIT_SSH_HOST": "github", + "EXTERNAL_GIT_PUBLIC_KEY_FINGERPRINT": "fingerprint", + "USE_EXTERNAL_CHECKS_DIR": "True", + "EXTERNAL_DIR_OWNER": "test", + "EXTERNAL_DIR_REPOSITORY": "repository", + "EXTERNAL_DIR_ASSET_NAME": "rules", + "RULES": "", + "APP_ID_GITHUB": "app_id", + "INSTALATION_ID_GITHUB": "installation_id" + }, + } + + + util = Utils() + result = util.configurate_external_checks("checkov",json_data,None, "ssh:2231231:123123") + + assert result is None diff --git a/tools/devsecops_engine_tools/engine_utilities/utils/session_manager.py b/tools/devsecops_engine_tools/engine_utilities/utils/session_manager.py index 9fbb67288..7564a7084 100644 --- a/tools/devsecops_engine_tools/engine_utilities/utils/session_manager.py +++ b/tools/devsecops_engine_tools/engine_utilities/utils/session_manager.py @@ -1,5 +1,5 @@ import requests - +from requests.adapters import HTTPAdapter class SessionManager: _instance = None @@ -11,4 +11,7 @@ def __new__(cls, token=None, host=None): cls._host = host if not cls._instance: cls._instance = requests.Session() + adapter = HTTPAdapter(pool_maxsize=40) + cls._instance.mount('https://', adapter) + cls._instance.mount('http://', adapter) return cls diff --git a/tools/devsecops_engine_tools/engine_utilities/utils/utils.py b/tools/devsecops_engine_tools/engine_utilities/utils/utils.py old mode 100644 new mode 100755 index 32d90b1cf..07603af73 --- a/tools/devsecops_engine_tools/engine_utilities/utils/utils.py +++ b/tools/devsecops_engine_tools/engine_utilities/utils/utils.py @@ -1,4 +1,28 @@ import zipfile +import tarfile +import platform +from devsecops_engine_tools.engine_utilities.github.infrastructure.github_api import ( + GithubApi, +) +from devsecops_engine_tools.engine_utilities.ssh.managment_private_key import ( + create_ssh_private_file, + add_ssh_private_key, + decode_base64, + config_knowns_hosts, +) +from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger +from devsecops_engine_tools.engine_utilities import settings +logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() +import base64 +import re + +from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold +from devsecops_engine_tools.engine_core.src.domain.model.level_vulnerability import ( + LevelVulnerability, +) +from devsecops_engine_tools.engine_core.src.domain.model.level_compliance import ( + LevelCompliance, +) class Utils: @@ -6,3 +30,114 @@ class Utils: def unzip_file(self, zip_file_path, extract_path): with zipfile.ZipFile(zip_file_path, "r") as zip_ref: zip_ref.extractall(extract_path) + + def extract_targz_file(self, tar_file_path, extract_path): + with tarfile.open(tar_file_path, "r:gz") as tar_ref: + tar_ref.extractall(path=extract_path) + + def configurate_external_checks(self, tool, config_tool, secret_tool, secret_external_checks, agent_work_folder="/tmp"): + try: + agent_env = None + secret = None + github_token = None + github_api = GithubApi() + + if secret_tool is not None: + secret = secret_tool + github_token = github_api.get_installation_access_token( + secret["github_token"], + config_tool[tool]["APP_ID_GITHUB"], + config_tool[tool]["INSTALLATION_ID_GITHUB"] + ) + + elif secret_external_checks is not None: + secret_external_checks_parts = { + "github_token": ( + secret_external_checks.split("github_token:")[1] + if "github_token" in secret_external_checks + else None + ), + "github_apps": ( + secret_external_checks.split("github_apps:")[1] + if "github_apps" in secret_external_checks + else None + ), + "repository_ssh_private_key": ( + secret_external_checks.split("ssh:")[1].split(":")[0] + if "ssh" in secret_external_checks + else None + ), + "repository_ssh_password": ( + secret_external_checks.split("ssh:")[1].split(":")[1] + if "ssh" in secret_external_checks + else None + ), + } + + secret = { + key: secret_external_checks_parts[key] + for key in secret_external_checks_parts + if secret_external_checks_parts[key] is not None + } + + if secret is None: + logger.warning("The secret is not configured for external controls") + + + elif config_tool[tool]["USE_EXTERNAL_CHECKS_GIT"] and platform.system() in ( + "Linux", "Darwin", + ): + config_knowns_hosts( + config_tool[tool]["EXTERNAL_GIT_SSH_HOST"], + config_tool[tool]["EXTERNAL_GIT_PUBLIC_KEY_FINGERPRINT"], + ) + ssh_key_content = decode_base64(secret["repository_ssh_private_key"]) + ssh_key_file_path = "/tmp/ssh_key_file" + create_ssh_private_file(ssh_key_file_path, ssh_key_content) + ssh_key_password = decode_base64(secret["repository_ssh_password"]) + agent_env = add_ssh_private_key(ssh_key_file_path, ssh_key_password) + + elif config_tool[tool]["USE_EXTERNAL_CHECKS_DIR"]: + if not github_token: + github_token = github_api.get_installation_access_token( + secret.get("github_apps"), + config_tool[tool]["APP_ID_GITHUB"], + config_tool[tool]["INSTALLATION_ID_GITHUB"] + ) if secret.get("github_apps") else secret.get("github_token") + github_api.download_latest_release_assets( + config_tool[tool]["EXTERNAL_DIR_OWNER"], + config_tool[tool]["EXTERNAL_DIR_REPOSITORY"], + github_token, + agent_work_folder + ) + + except Exception as ex: + logger.error(f"An error occurred configuring external checks: {ex}") + return agent_env + + def encode_token_to_base64(self, token): + token_bytes = f"{token}:".encode("utf-8") + base64_token = base64.b64encode(token_bytes).decode("utf-8") + return base64_token + + def update_threshold(self, threshold: Threshold, exclusions_data, pipeline_name): + def set_threshold(new_threshold): + threshold.vulnerability = LevelVulnerability(new_threshold.get("VULNERABILITY")) + threshold.compliance = LevelCompliance(new_threshold.get("COMPLIANCE")) if new_threshold.get("COMPLIANCE") else threshold.compliance + threshold.cve = new_threshold.get("CVE") if new_threshold.get("CVE") is not None else threshold.cve + return threshold + + threshold_pipeline = exclusions_data.get(pipeline_name, {}).get("THRESHOLD", {}) + if threshold_pipeline: + return set_threshold(threshold_pipeline) + + search_patterns = exclusions_data.get("BY_PATTERN_SEARCH", {}) + + match_pattern = next( + (v["THRESHOLD"] + for pattern, v in search_patterns.items() + if re.match(pattern, pipeline_name, re.IGNORECASE)), + None + ) + + return set_threshold(match_pattern) if match_pattern else threshold diff --git a/tools/devsecops_engine_tools/version.py b/tools/devsecops_engine_tools/version.py index 72cadbadb..c75171312 100644 --- a/tools/devsecops_engine_tools/version.py +++ b/tools/devsecops_engine_tools/version.py @@ -1 +1 @@ -version = '1.11.5' +version = '1.32.1' diff --git a/tools/requirements.txt b/tools/requirements.txt index f8d7a65b9..ba00810ed 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -12,4 +12,10 @@ PyGithub==2.3.0 distro==1.9.0 boto3==1.34.157 docker==7.1.0 -setuptools==72.1.0 \ No newline at end of file +setuptools==72.1.0 +rich==13.9.4 +cpe==1.3.1 +packageurl-python==0.15.6 +ruamel.yaml==0.18.6 +Authlib==1.3.2 +PyJWT==2.9.0 diff --git a/tools/requirements_test.txt b/tools/requirements_test.txt index f95ba0739..4a51aea9f 100644 --- a/tools/requirements_test.txt +++ b/tools/requirements_test.txt @@ -4,4 +4,7 @@ coverage_badge==1.1.2 flake8==7.1.1 black==24.8.0 pre-commit==3.5.0 -tabulate==0.9.0 \ No newline at end of file +tabulate==0.9.0 +ruamel.yaml==0.18.6 +Authlib==1.3.2 +PyJWT==2.9.0 \ No newline at end of file diff --git a/tools/setup.py b/tools/setup.py index e89c1b57e..d77e458ba 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -33,7 +33,8 @@ def get_requirements(): packages=find_packages(exclude=["**test**"]), entry_points={ 'console_scripts': [ - 'devsecops-engine-tools=devsecops_engine_tools.engine_core.src.applications.runner_engine_core:application_core' + 'devsecops-engine-tools=devsecops_engine_tools.engine_core.src.applications.runner_engine_core:application_core', + 'devsecops-engine-tools.report-sonar=devsecops_engine_tools.engine_utilities.sonarqube.src.applications.runner_report_sonar:runner_report_sonar' ] }, classifiers=[