From e4a9ac801ffac7fdbf388614adbb74923f668d7a Mon Sep 17 00:00:00 2001 From: reichertan Date: Fri, 9 Feb 2024 20:36:44 +0100 Subject: [PATCH 1/3] Sonatype parser improved --- dojo/settings/settings.dist.py | 2 + dojo/tools/sonatype/identifier.py | 53 +++++ dojo/tools/sonatype/parser.py | 213 +++++++----------- unittests/scans/sonatype/many_vulns.json | 11 + unittests/scans/sonatype/no_vuln.json | 52 +++++ .../{one_vuln.json => two_vulns.json} | 0 unittests/tools/test_sonatype_parser.py | 70 +++++- 7 files changed, 260 insertions(+), 141 deletions(-) create mode 100644 dojo/tools/sonatype/identifier.py create mode 100644 unittests/scans/sonatype/no_vuln.json rename unittests/scans/sonatype/{one_vuln.json => two_vulns.json} (100%) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index fad2454b7ca..0d4283dbf26 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -1212,6 +1212,7 @@ def saml2_attrib_map_format(dict): 'Cloudsploit Scan': ['title', 'description'], 'SonarQube Scan': ['cwe', 'severity', 'file_path'], 'SonarQube API Import': ['title', 'file_path', 'line'], + 'Sonatype Application Scan': ['title', 'cwe', 'file_path', 'component_name', 'component_version', 'vulnerability_ids'], 'Dependency Check Scan': ['title', 'cwe', 'file_path'], 'Dockle Scan': ['title', 'description', 'vuln_id_from_tool'], 'Dependency Track Finding Packaging Format (FPF) Export': ['component_name', 'component_version', 'vulnerability_ids'], @@ -1400,6 +1401,7 @@ def saml2_attrib_map_format(dict): 'SonarQube Scan detailed': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL, 'SonarQube Scan': DEDUPE_ALGO_HASH_CODE, 'SonarQube API Import': DEDUPE_ALGO_HASH_CODE, + 'Sonatype Application Scan': DEDUPE_ALGO_HASH_CODE, 'Dependency Check Scan': DEDUPE_ALGO_HASH_CODE, 'Dockle Scan': DEDUPE_ALGO_HASH_CODE, 'Tenable Scan': DEDUPE_ALGO_HASH_CODE, diff --git a/dojo/tools/sonatype/identifier.py b/dojo/tools/sonatype/identifier.py new file mode 100644 index 00000000000..764cf03b97f --- /dev/null +++ b/dojo/tools/sonatype/identifier.py @@ -0,0 +1,53 @@ +# Implemented according to Sonatype Component Identifiers +# https://help.sonatype.com/en/referencing-package-url--purl--and-component-identifiers.html +class ComponentIdentifier: + + def __init__(self, component): + self._component_id = "" + self._component_name = "" + self._component_version = "" + + if "componentIdentifier" in component: + component_coordinates = component["componentIdentifier"]["coordinates"] + componant_format = component["componentIdentifier"]["format"] + + if componant_format in ["a-name", "pypi", "rpm", "gem", "golang", "conan", "conda", "bower", "composer", + "cran", "cargo", "cocoapods", "drupal", "pecoff", "swift", "generic", + "operating-system"]: + self.set_name_version_component(component_coordinates) + elif componant_format == "maven": + self.set_maven_component(component_coordinates) + elif componant_format in ["npm", "nuget"]: + self.set_package_id_version_component(component_coordinates) + elif "displayName" in component: + self._component_id = component["displayName"] + self._component_name = component["displayName"] + + @property + def component_id(self): + return self._component_id + + @property + def component_name(self): + return self._component_name + + @property + def component_version(self): + return self._component_version + + def set_name_version_component(self, component_coordinates): + self._component_id = f"{component_coordinates['name']} {component_coordinates['version']}" + self._component_name = component_coordinates["name"] + self._component_version = component_coordinates["version"] + + def set_maven_component(self, component_coordinates): + self._component_id = (f"{component_coordinates['artifactId']} " + f"{component_coordinates['groupId']} " + f"{component_coordinates['version']}") + self._component_name = component_coordinates["artifactId"] + self._component_version = component_coordinates["version"] + + def set_package_id_version_component(self, component_coordinates): + self._component_id = f"{component_coordinates['packageId']} {component_coordinates['version']}" + self._component_name = component_coordinates["packageId"] + self._component_version = component_coordinates["version"] diff --git a/dojo/tools/sonatype/parser.py b/dojo/tools/sonatype/parser.py index 0e3934f9131..bc60b536bec 100644 --- a/dojo/tools/sonatype/parser.py +++ b/dojo/tools/sonatype/parser.py @@ -1,154 +1,95 @@ -""" Surely a fragile parser, but gets things started and will evolve over time I guess. -It seems that a lot of the json data does not have any securityData data -So these are just skipped, since there is nothing much to do with them here -""" import json - from dojo.models import Finding +from dojo.tools.sonatype.identifier import ComponentIdentifier -class SonatypeParser(object): +class SonatypeParser: # This parser does not deal with licenses information. def get_scan_types(self): return ["Sonatype Application Scan"] def get_label_for_scan_types(self, scan_type): - return scan_type # no custom label for now + return "Sonatype Application Scan" def get_description_for_scan_types(self, scan_type): return "Can be imported in JSON format" def get_findings(self, json_output, test): - tree = json.load(json_output) - return self.get_items(tree, test) - - def get_items(self, tree, test): - items = {} - if "components" in tree: - vulnerability_tree = tree["components"] + sonatype_report = json.load(json_output) + findings = [] + if "components" in sonatype_report: + components = sonatype_report["components"] - for node in vulnerability_tree: - item = get_item(node, test) - if item is None: + for component in components: + if component["securityData"] is None or len(component["securityData"]["securityIssues"]) < 1: continue - # TODO - unique_key = node["hash"] - items[unique_key] = item - - return list(items.values()) - - -def get_item(vulnerability, test): - # Following the CVSS Scoring per https://nvd.nist.gov/vuln-metrics/cvss - if ( - vulnerability["securityData"] is not None - and len(vulnerability["securityData"]["securityIssues"]) >= 1 - ): - # there can be nothing in the array, or securityData can be null altogether. If the latter, well, nothing much to do? - # issues is an array, and there can be 2+ of them, e.g. a cve and a sonatype entry or two cves - # Given the current Finding class, if a cve, will be the main. If not a cve, then CVE ref will remain null due to regex. - # Others go to references. - main_finding = vulnerability["securityData"]["securityIssues"][0] - - if main_finding.get("source") == "cve": - vulnerability_id = main_finding.get("reference") - else: - # if sonatype of else, will not match Finding model today - vulnerability_id = None - - if main_finding["severity"] <= 3.9: - severity = "Low" - elif ( - main_finding["severity"] > 4.0 and main_finding["severity"] <= 6.9 - ): - severity = "Medium" - elif main_finding["severity"] and main_finding["severity"] <= 8.9: - severity = "High" - else: - severity = "Critical" - - references = [] - if len(vulnerability["securityData"]["securityIssues"]) > 1: - for additional_issue in vulnerability["securityData"][ - "securityIssues" - ]: - references.append( - "{}, {}, {}, {}, {} ".format( - additional_issue.get("reference"), - additional_issue.get("status"), - additional_issue.get("severity"), - additional_issue.get("threatCategory"), - additional_issue.get("url"), - ) - ) - - component_id = "" - if "componentIdentifier" in vulnerability: - if vulnerability["componentIdentifier"]["format"] == "maven": - component_id = "{} {} {}".format( - vulnerability["componentIdentifier"]["coordinates"][ - "artifactId" - ], - vulnerability["componentIdentifier"]["coordinates"][ - "groupId" - ], - vulnerability["componentIdentifier"]["coordinates"][ - "version" - ], - ) - elif vulnerability["componentIdentifier"]["format"] == "a-name": - component_id = "{} {} {}".format( - vulnerability["componentIdentifier"]["coordinates"][ - "name" - ], - vulnerability["componentIdentifier"]["coordinates"][ - "qualifier" - ], - vulnerability["componentIdentifier"]["coordinates"][ - "version" - ], - ) - - finding_title = "{} - {}".format( - main_finding["reference"], component_id - ) - - finding_description = "Hash {}\n\n".format(vulnerability["hash"]) - finding_description += component_id - finding_description += '\n\nPlease check the CVE details of this finding for a detailed description. The details of issues beginning with "SONATYPE-" can be found by contacting Sonatype, Inc. or through mechanisms they have provided in their product.' - threat_category = main_finding.get( - "threatCategory", "CVSS vector not provided. " - ).title() - status = main_finding["status"] - main_finding.get("severity", "No CVSS score yet.") - if "pathnames" in vulnerability: - file_path = " ".join(vulnerability["pathnames"])[:1000] - else: - file_path = "" - - # create the finding object - finding = Finding( - title=finding_title, - test=test, - severity=severity, - description=finding_description, - mitigation=status, - references="{}\n{}\n".format( - main_finding["url"], "\n".join(references) - ), - false_p=False, - duplicate=False, - out_of_scope=False, - mitigated=None, - file_path=file_path, - impact=threat_category, - static_finding=True - ) - if vulnerability_id: - finding.unsaved_vulnerability_ids = [vulnerability_id] - finding.description = finding.description.strip() - - return finding + + for security_issue in component["securityData"]["securityIssues"]: + finding = get_finding(security_issue, component, test) + findings.append(finding) + + return findings + + +def get_finding(security_issue, component, test): + + severity = get_severity(security_issue) + threat_category = security_issue.get("threatCategory", "CVSS vector not provided. ").title() + status = security_issue["status"] + reference = security_issue["url"] + + identifier = ComponentIdentifier(component) + title = f"{security_issue['reference']} - {identifier.component_id}" + + finding_description = f"Hash {component['hash']}\n\n" + finding_description += identifier.component_id + finding_description = finding_description.strip() + + finding = Finding( + test=test, + title=title, + description=finding_description, + component_name=identifier.component_name, + component_version=identifier.component_version, + severity=severity, + mitigation=status, + references=reference, + impact=threat_category, + static_finding=True + ) + if "cwe" in security_issue: + finding.cwe = security_issue["cwe"] + + if "cvssVector" in security_issue: + finding.cvssv3 = security_issue["cvssVector"] + + if "pathnames" in component: + finding.file_path = " ".join(component["pathnames"])[:1000] + + if security_issue.get("source") == "cve": + vulnerability_id = security_issue.get("reference") + finding.unsaved_vulnerability_ids = [vulnerability_id] + + if status == "Open": + pass + elif status == "Acknowledged": + finding.verified = True + elif status == "Not Applicable": + finding.verified = True + finding.out_of_scope = True + elif status == "Confirmed": + finding.verified = True + + return finding + + +def get_severity(vulnerability): + if vulnerability["severity"] <= 3.9: + return "Low" + elif vulnerability["severity"] <= 6.9: + return "Medium" + elif vulnerability["severity"] <= 8.9: + return "High" else: - return None + return "Critical" diff --git a/unittests/scans/sonatype/many_vulns.json b/unittests/scans/sonatype/many_vulns.json index 05b986aec07..bfe2b37026b 100644 --- a/unittests/scans/sonatype/many_vulns.json +++ b/unittests/scans/sonatype/many_vulns.json @@ -295,6 +295,17 @@ "status": "Open", "url": null, "threatCategory": "severe" + }, + { + "source": "sonatype", + "reference": "sonatype-2023-4856", + "severity": 5.3, + "status": "Not Applicable", + "url": null, + "threatCategory": "severe", + "cwe": "693", + "cvssVector": "CVSS:3.1/AV:N/AC:H/PR:N/UI:R/S:U/C:N/I:H/A:N", + "cvssVectorSource": "sonatype_cvss_3" } ] } diff --git a/unittests/scans/sonatype/no_vuln.json b/unittests/scans/sonatype/no_vuln.json new file mode 100644 index 00000000000..f472afd557e --- /dev/null +++ b/unittests/scans/sonatype/no_vuln.json @@ -0,0 +1,52 @@ +{ + "components": [ + { + "hash": "212387a39088ecd3daff", + "componentIdentifier": { + "format": "maven", + "coordinates": { + "artifactId": "okhttp", + "classifier": "", + "extension": "jar", + "groupId": "com.squareup.okhttp", + "version": "2.6.0" + } + }, + "proprietary": false, + "matchState": "exact", + "pathnames": [ + "WEB-INF/lib/okhttp-2.6.0.jar" + ], + "licenseData": { + "declaredLicenses": [ + { + "licenseId": "Apache-2.0", + "licenseName": "Apache-2.0" + } + ], + "observedLicenses": [ + { + "licenseId": "Apache-2.0", + "licenseName": "Apache-2.0" + } + ], + "overriddenLicenses": [], + "status": "Open", + "effectiveLicenseThreats": [ + { + "licenseThreatGroupName": "Liberal", + "licenseThreatGroupLevel": 0, + "licenseThreatGroupCategory": "no-threat" + } + ] + }, + "securityData": { + "securityIssues": [] + } + } + ], + "matchSummary": { + "totalComponentCount": 1011, + "knownComponentCount": 826 + } +} diff --git a/unittests/scans/sonatype/one_vuln.json b/unittests/scans/sonatype/two_vulns.json similarity index 100% rename from unittests/scans/sonatype/one_vuln.json rename to unittests/scans/sonatype/two_vulns.json diff --git a/unittests/tools/test_sonatype_parser.py b/unittests/tools/test_sonatype_parser.py index a3bfcf2a13c..c898458a059 100644 --- a/unittests/tools/test_sonatype_parser.py +++ b/unittests/tools/test_sonatype_parser.py @@ -4,12 +4,12 @@ class TestSonatypeParser(DojoTestCase): - def test_parse_file_with_one_vuln(self): - testfile = open("unittests/scans/sonatype/one_vuln.json") + def test_parse_file_with_two_vulns(self): + testfile = open("unittests/scans/sonatype/two_vulns.json") parser = SonatypeParser() findings = parser.get_findings(testfile, Test()) testfile.close() - self.assertEqual(1, len(findings)) + self.assertEqual(2, len(findings)) self.assertEqual(1, len(findings[0].unsaved_vulnerability_ids)) self.assertEqual("CVE-2016-2402", findings[0].unsaved_vulnerability_ids[0]) @@ -18,11 +18,71 @@ def test_parse_file_with_many_vulns(self): parser = SonatypeParser() findings = parser.get_findings(testfile, Test()) testfile.close() - self.assertEqual(3, len(findings)) + self.assertEqual(6, len(findings)) def test_parse_file_with_long_file_path(self): testfile = open("unittests/scans/sonatype/long_file_path.json") parser = SonatypeParser() findings = parser.get_findings(testfile, Test()) testfile.close() - self.assertEqual(2, len(findings)) + self.assertEqual(3, len(findings)) + + def test_find_no_vuln(self): + testfile = open("unittests/scans/sonatype/no_vuln.json") + parser = SonatypeParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(0, len(findings)) + + def test_component_parsed_correctly(self): + testfile = open("unittests/scans/sonatype/many_vulns.json") + parser = SonatypeParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual("sonatype-2023-4856 - okhttp com.squareup.okhttp 2.6.0", findings[5].title) + self.assertEqual("okhttp", findings[5].component_name) + self.assertEqual("2.6.0", findings[5].component_version) + + def test_severity_parsed_correctly(self): + testfile = open("unittests/scans/sonatype/many_vulns.json") + parser = SonatypeParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual("Medium", findings[0].severity) + self.assertEqual("High", findings[1].severity) + self.assertEqual("High", findings[2].severity) + self.assertEqual("Medium", findings[3].severity) + self.assertEqual("Medium", findings[4].severity) + self.assertEqual("Medium", findings[5].severity) + + def test_cwe_parsed_correctly(self): + testfile = open("unittests/scans/sonatype/many_vulns.json") + parser = SonatypeParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual("693", findings[5].cwe) + + def test_cvssv3_parsed_correctly(self): + testfile = open("unittests/scans/sonatype/many_vulns.json") + parser = SonatypeParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual("CVSS:3.1/AV:N/AC:H/PR:N/UI:R/S:U/C:N/I:H/A:N", findings[5].cvssv3) + + def test_filepath_parsed_correctly(self): + testfile = open("unittests/scans/sonatype/many_vulns.json") + parser = SonatypeParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual("WEB-INF/lib/okhttp-2.6.0.jar", findings[5].file_path) + + def test_status_parsed_correctly(self): + testfile = open("unittests/scans/sonatype/many_vulns.json") + parser = SonatypeParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(False, findings[4].verified) + self.assertEqual(False, findings[4].out_of_scope) + self.assertEqual(True, findings[5].verified) + self.assertEqual(True, findings[5].out_of_scope) + From 2f0a13432234f5e274e332ca1c6b2a6595224870 Mon Sep 17 00:00:00 2001 From: reichertan <70580399+reichertan@users.noreply.github.com> Date: Fri, 9 Feb 2024 21:33:42 +0100 Subject: [PATCH 2/3] Blank line at end of file removed. --- unittests/tools/test_sonatype_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/unittests/tools/test_sonatype_parser.py b/unittests/tools/test_sonatype_parser.py index c898458a059..a84edfa3e34 100644 --- a/unittests/tools/test_sonatype_parser.py +++ b/unittests/tools/test_sonatype_parser.py @@ -85,4 +85,3 @@ def test_status_parsed_correctly(self): self.assertEqual(False, findings[4].out_of_scope) self.assertEqual(True, findings[5].verified) self.assertEqual(True, findings[5].out_of_scope) - From 17cfe853ef67a5d7612f261a5d645fef63fdafaf Mon Sep 17 00:00:00 2001 From: reichertan <70580399+reichertan@users.noreply.github.com> Date: Sat, 10 Feb 2024 09:25:45 +0100 Subject: [PATCH 3/3] Sonatype status evaluation removed. --- dojo/tools/sonatype/parser.py | 10 ---------- unittests/tools/test_sonatype_parser.py | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/dojo/tools/sonatype/parser.py b/dojo/tools/sonatype/parser.py index bc60b536bec..5458f64a9f7 100644 --- a/dojo/tools/sonatype/parser.py +++ b/dojo/tools/sonatype/parser.py @@ -71,16 +71,6 @@ def get_finding(security_issue, component, test): vulnerability_id = security_issue.get("reference") finding.unsaved_vulnerability_ids = [vulnerability_id] - if status == "Open": - pass - elif status == "Acknowledged": - finding.verified = True - elif status == "Not Applicable": - finding.verified = True - finding.out_of_scope = True - elif status == "Confirmed": - finding.verified = True - return finding diff --git a/unittests/tools/test_sonatype_parser.py b/unittests/tools/test_sonatype_parser.py index a84edfa3e34..48e4b4b056e 100644 --- a/unittests/tools/test_sonatype_parser.py +++ b/unittests/tools/test_sonatype_parser.py @@ -75,13 +75,3 @@ def test_filepath_parsed_correctly(self): findings = parser.get_findings(testfile, Test()) testfile.close() self.assertEqual("WEB-INF/lib/okhttp-2.6.0.jar", findings[5].file_path) - - def test_status_parsed_correctly(self): - testfile = open("unittests/scans/sonatype/many_vulns.json") - parser = SonatypeParser() - findings = parser.get_findings(testfile, Test()) - testfile.close() - self.assertEqual(False, findings[4].verified) - self.assertEqual(False, findings[4].out_of_scope) - self.assertEqual(True, findings[5].verified) - self.assertEqual(True, findings[5].out_of_scope)