From f35fc168bf044d9c1b67d946ed02f1c55fa3aeb1 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Thu, 18 Jan 2024 00:01:05 +0100 Subject: [PATCH 1/6] :sparkles: merge clair and clair klar together --- .../en/integrations/parsers/file/clair.md | 5 +- .../integrations/parsers/file/clair_klar.md | 6 - dojo/tools/clair/parser.py | 215 +++++++++++++----- dojo/tools/clair_klar/__init__.py | 0 dojo/tools/clair_klar/parser.py | 134 ----------- .../clair/{empty.json => clair_empty.json} | 0 .../{few_vuln.json => clair_few_vuln.json} | 0 .../{many_vul.json => clair_many_vul.json} | 0 .../empty.json => clair/clairklar_empty.json} | 0 .../high.json => clair/clairklar_high.json} | 0 .../mixed.json => clair/clairklar_mixed.json} | 0 unittests/tools/test_clair_klar_parser.py | 35 --- unittests/tools/test_clair_parser.py | 36 ++- 13 files changed, 196 insertions(+), 235 deletions(-) delete mode 100644 docs/content/en/integrations/parsers/file/clair_klar.md delete mode 100644 dojo/tools/clair_klar/__init__.py delete mode 100644 dojo/tools/clair_klar/parser.py rename unittests/scans/clair/{empty.json => clair_empty.json} (100%) rename unittests/scans/clair/{few_vuln.json => clair_few_vuln.json} (100%) rename unittests/scans/clair/{many_vul.json => clair_many_vul.json} (100%) rename unittests/scans/{clair_klar/empty.json => clair/clairklar_empty.json} (100%) rename unittests/scans/{clair_klar/high.json => clair/clairklar_high.json} (100%) rename unittests/scans/{clair_klar/mixed.json => clair/clairklar_mixed.json} (100%) delete mode 100644 unittests/tools/test_clair_klar_parser.py diff --git a/docs/content/en/integrations/parsers/file/clair.md b/docs/content/en/integrations/parsers/file/clair.md index e2e9bd6dc5a..6e7a0215ed5 100644 --- a/docs/content/en/integrations/parsers/file/clair.md +++ b/docs/content/en/integrations/parsers/file/clair.md @@ -2,4 +2,7 @@ title: "Clair Scan" toc_hide: true --- -Import JSON reports of Docker image vulnerabilities. +Import JSON reports of Docker image vulnerabilities from clair or the clair klar client. + +### Sample Scan Data +Sample Clair and Clair Klar scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/clair). \ No newline at end of file diff --git a/docs/content/en/integrations/parsers/file/clair_klar.md b/docs/content/en/integrations/parsers/file/clair_klar.md deleted file mode 100644 index 4328a17bda8..00000000000 --- a/docs/content/en/integrations/parsers/file/clair_klar.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: "Clair Klar Scan" -toc_hide: true ---- -Import JSON reports of Docker image vulnerabilities from clair klar -client. \ No newline at end of file diff --git a/dojo/tools/clair/parser.py b/dojo/tools/clair/parser.py index dbf30f4e987..dd1aef59cfd 100644 --- a/dojo/tools/clair/parser.py +++ b/dojo/tools/clair/parser.py @@ -1,6 +1,7 @@ import json - +import logging from dojo.models import Finding +logger = logging.getLogger(__name__) class ClairParser(object): @@ -11,75 +12,179 @@ def get_label_for_scan_types(self, scan_type): return scan_type # no custom label for now def get_description_for_scan_types(self, scan_type): - return "Import JSON reports of Docker image vulnerabilities." + return "Import JSON reports of Docker image vulnerabilities from clair or clair klar client." def get_findings(self, json_output, test): tree = self.parse_json(json_output) - return self.get_items(tree, test) + if tree: + if self.scanner == "clair": + return self.get_items_clair(tree, test) + elif self.scanner == "clairklar": + items = list() + clair_severities = [ + "Unknown", + "Negligible", + "Low", + "Medium", + "High", + "Critical", + "Defcon1", + ] + for clair_severity in clair_severities: + items.extend( + self.set_items_for_severity(tree, test, clair_severity) + ) + return items + else: + return list() def parse_json(self, json_output): - data = json_output.read() try: - tree = json.loads(str(data, "utf-8")) + data = json_output.read() + try: + tree = json.loads(str(data, "utf-8")) + except BaseException: + tree = json.loads(data) + if tree.get('image'): + self.scanner = "clair" + subtree = tree.get("vulnerabilities") + elif tree.get('LayerCount'): + self.scanner = "clairklar" + subtree = tree.get("Vulnerabilities") except BaseException: - tree = json.loads(data) - return tree.get("vulnerabilities") + raise ValueError("Invalid format") + return subtree - def get_items(self, tree, test): - items = {} + def set_items_for_severity(self, tree, test, severity): + items = list() + tree_severity = tree.get(severity) + if tree_severity: + for data in self.get_items_clairklar(tree_severity, test): + items.append(data) + logger.debug("Appended findings for severity " + severity) + else: + logger.debug("No findings for severity " + severity) + return items + def get_items_clair(self, tree, test): + items = {} for node in tree: - item = get_item(node, test) + item = self.get_item_clair(node, test) unique_key = str(node["vulnerability"]) + str(node["featurename"]) items[unique_key] = item - return list(items.values()) + def get_item_clair(self, item_node, test): + if ( + item_node["severity"] == "Negligible" + or item_node["severity"] == "Unknown" + ): + severity = "Info" + else: + severity = item_node["severity"] + + finding = Finding( + title=item_node["vulnerability"] + + " - " + + "(" + + item_node["featurename"] + + ", " + + item_node["featureversion"] + + ")", + test=test, + severity=severity, + description=item_node["description"] + + "\n Vulnerable feature: " + + item_node["featurename"] + + "\n Vulnerable Versions: " + + str(item_node["featureversion"]) + + "\n Fixed by: " + + str(item_node["fixedby"]) + + "\n Namespace: " + + str(item_node["namespace"]) + + "\n CVE: " + + str(item_node["vulnerability"]), + mitigation=item_node["fixedby"], + references=item_node["link"], + component_name=item_node["featurename"], + component_version=item_node["featureversion"], + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated=None, + static_finding=True, + dynamic_finding=False, + impact="No impact provided", + ) + if item_node["vulnerability"]: + finding.unsaved_vulnerability_ids = [item_node["vulnerability"]] + return finding + + def get_items_clairklar(self, tree_severity, test): + items = {} + for node in tree_severity: + item = self.get_item_clairklar(node, test) + unique_key = str(node["Name"]) + str(node["FeatureName"]) + items[unique_key] = item + return items.values() -def get_item(item_node, test): - if ( - item_node["severity"] == "Negligible" - or item_node["severity"] == "Unknown" - ): - severity = "Info" - else: - severity = item_node["severity"] + def get_item_clairklar(self, item_node, test): + if item_node["Severity"] == "Negligible": + severity = "Info" + elif item_node["Severity"] == "Unknown": + severity = "Critical" + elif item_node["Severity"] == "Defcon1": + severity = "Critical" + else: + severity = item_node["Severity"] + description = "" + if "Description" in item_node: + description += item_node["Description"] + "\n
" + if "FeatureName" in item_node: + description += ( + "Vulnerable feature: " + item_node["FeatureName"] + "\n
" + ) + if "FeatureVersion" in item_node: + description += " Vulnerable Versions: " + str( + item_node["FeatureVersion"] + ) - finding = Finding( - title=item_node["vulnerability"] - + " - " - + "(" - + item_node["featurename"] - + ", " - + item_node["featureversion"] - + ")", - test=test, - severity=severity, - description=item_node["description"] - + "\n Vulnerable feature: " - + item_node["featurename"] - + "\n Vulnerable Versions: " - + str(item_node["featureversion"]) - + "\n Fixed by: " - + str(item_node["fixedby"]) - + "\n Namespace: " - + str(item_node["namespace"]) - + "\n CVE: " - + str(item_node["vulnerability"]), - mitigation=item_node["fixedby"], - references=item_node["link"], - component_name=item_node["featurename"], - component_version=item_node["featureversion"], - false_p=False, - duplicate=False, - out_of_scope=False, - mitigated=None, - static_finding=True, - dynamic_finding=False, - impact="No impact provided", - ) + mitigation = "" + if "FixedBy" in item_node: + description = description + "\n Fixed by: " + str(item_node["FixedBy"]) + mitigation = ( + "Please use version " + + item_node["FixedBy"] + + " of library " + + item_node["FeatureName"] + ) + else: + mitigation = "A patch could not been found" - if item_node["vulnerability"]: - finding.unsaved_vulnerability_ids = [item_node["vulnerability"]] + link = "" + if "Link" in item_node: + link = item_node["Link"] - return finding + finding = Finding( + title=item_node["Name"] + + " - " + + "(" + + item_node["FeatureName"] + + ", " + + item_node["FeatureVersion"] + + ")", + test=test, + severity=severity, + description=description, + mitigation=mitigation, + references=link, + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated=None, + cwe=1035, # Vulnerable Third Party Component + static_finding=True, + dynamic_finding=False, + impact="No impact provided", + ) + return finding diff --git a/dojo/tools/clair_klar/__init__.py b/dojo/tools/clair_klar/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dojo/tools/clair_klar/parser.py b/dojo/tools/clair_klar/parser.py deleted file mode 100644 index 2b5feafea87..00000000000 --- a/dojo/tools/clair_klar/parser.py +++ /dev/null @@ -1,134 +0,0 @@ -import json -import logging - -from dojo.models import Finding - -logger = logging.getLogger(__name__) - - -class ClairKlarParser(object): - def get_scan_types(self): - return ["Clair Klar Scan"] - - def get_label_for_scan_types(self, scan_type): - return scan_type # no custom label for now - - def get_description_for_scan_types(self, scan_type): - return "Import JSON reports of Docker image vulnerabilities from clair klar client." - - def get_findings(self, json_output, test): - tree = self.parse_json(json_output) - - items = list() - clair_severities = [ - "Unknown", - "Negligible", - "Low", - "Medium", - "High", - "Critical", - "Defcon1", - ] - if tree: - for clair_severity in clair_severities: - items.extend( - self.set_items_for_severity(tree, test, clair_severity) - ) - return items - - def parse_json(self, json_output): - try: - data = json_output.read() - try: - tree = json.loads(str(data, "utf-8")) - except BaseException: - tree = json.loads(data) - subtree = tree.get("Vulnerabilities") - except BaseException: - raise ValueError("Invalid format") - - return subtree - - def set_items_for_severity(self, tree, test, severity): - items = list() - tree_severity = tree.get(severity) - if tree_severity: - for data in self.get_items(tree_severity, test): - items.append(data) - logger.debug("Appended findings for severity " + severity) - else: - logger.debug("No findings for severity " + severity) - return items - - def get_items(self, tree_severity, test): - items = {} - - for node in tree_severity: - item = get_item(node, test) - unique_key = str(node["Name"]) + str(node["FeatureName"]) - items[unique_key] = item - - return items.values() - - -def get_item(item_node, test): - if item_node["Severity"] == "Negligible": - severity = "Info" - elif item_node["Severity"] == "Unknown": - severity = "Critical" - elif item_node["Severity"] == "Defcon1": - severity = "Critical" - else: - severity = item_node["Severity"] - description = "" - if "Description" in item_node: - description += item_node["Description"] + "\n
" - if "FeatureName" in item_node: - description += ( - "Vulnerable feature: " + item_node["FeatureName"] + "\n
" - ) - if "FeatureVersion" in item_node: - description += " Vulnerable Versions: " + str( - item_node["FeatureVersion"] - ) - - mitigation = "" - if "FixedBy" in item_node: - description = description + "\n Fixed by: " + str(item_node["FixedBy"]) - mitigation = ( - "Please use version " - + item_node["FixedBy"] - + " of library " - + item_node["FeatureName"] - ) - else: - mitigation = "A patch could not been found" - - link = "" - if "Link" in item_node: - link = item_node["Link"] - - finding = Finding( - title=item_node["Name"] - + " - " - + "(" - + item_node["FeatureName"] - + ", " - + item_node["FeatureVersion"] - + ")", - test=test, - severity=severity, - description=description, - mitigation=mitigation, - references=link, - false_p=False, - duplicate=False, - out_of_scope=False, - mitigated=None, - cwe=1035, # Vulnerable Third Party Component - static_finding=True, - dynamic_finding=False, - impact="No impact provided", - ) - - return finding diff --git a/unittests/scans/clair/empty.json b/unittests/scans/clair/clair_empty.json similarity index 100% rename from unittests/scans/clair/empty.json rename to unittests/scans/clair/clair_empty.json diff --git a/unittests/scans/clair/few_vuln.json b/unittests/scans/clair/clair_few_vuln.json similarity index 100% rename from unittests/scans/clair/few_vuln.json rename to unittests/scans/clair/clair_few_vuln.json diff --git a/unittests/scans/clair/many_vul.json b/unittests/scans/clair/clair_many_vul.json similarity index 100% rename from unittests/scans/clair/many_vul.json rename to unittests/scans/clair/clair_many_vul.json diff --git a/unittests/scans/clair_klar/empty.json b/unittests/scans/clair/clairklar_empty.json similarity index 100% rename from unittests/scans/clair_klar/empty.json rename to unittests/scans/clair/clairklar_empty.json diff --git a/unittests/scans/clair_klar/high.json b/unittests/scans/clair/clairklar_high.json similarity index 100% rename from unittests/scans/clair_klar/high.json rename to unittests/scans/clair/clairklar_high.json diff --git a/unittests/scans/clair_klar/mixed.json b/unittests/scans/clair/clairklar_mixed.json similarity index 100% rename from unittests/scans/clair_klar/mixed.json rename to unittests/scans/clair/clairklar_mixed.json diff --git a/unittests/tools/test_clair_klar_parser.py b/unittests/tools/test_clair_klar_parser.py deleted file mode 100644 index f8478e2f59c..00000000000 --- a/unittests/tools/test_clair_klar_parser.py +++ /dev/null @@ -1,35 +0,0 @@ -from ..dojo_test_case import DojoTestCase -from dojo.tools.clair_klar.parser import ClairKlarParser - - -class TestFile(object): - def read(self): - return self.content - - def __init__(self, name, content): - self.name = name - self.content = content - - -class TestClairKlarParser(DojoTestCase): - - def test_parse_no_content_no_findings(self): - my_file_handle = open("unittests/scans/clair_klar/empty.json") - parser = ClairKlarParser() - findings = parser.get_findings(my_file_handle, None) - my_file_handle.close() - self.assertEqual(0, len(findings)) - - def test_high_findings(self): - my_file_handle = open("unittests/scans/clair_klar/high.json") - parser = ClairKlarParser() - findings = parser.get_findings(my_file_handle, None) - my_file_handle.close() - self.assertEqual(6, len(findings)) - - def test_mixed_findings(self): - my_file_handle = open("unittests/scans/clair_klar/mixed.json") - parser = ClairKlarParser() - findings = parser.get_findings(my_file_handle, None) - my_file_handle.close() - self.assertEqual(6, len(findings)) diff --git a/unittests/tools/test_clair_parser.py b/unittests/tools/test_clair_parser.py index b2d0b4b65be..31297c4df7f 100644 --- a/unittests/tools/test_clair_parser.py +++ b/unittests/tools/test_clair_parser.py @@ -4,15 +4,22 @@ class TestClairParser(DojoTestCase): - def test_no_findings(self): - my_file_handle = open("unittests/scans/clair/empty.json") + def test_no_findings_clair(self): + my_file_handle = open("unittests/scans/clair/clair_empty.json") parser = ClairParser() findings = parser.get_findings(my_file_handle, None) my_file_handle.close() self.assertEqual(0, len(findings)) - def test_many_findings(self): - my_file_handle = open("unittests/scans/clair/many_vul.json") + def test_few_findings_clair(self): + my_file_handle = open("unittests/scans/clair/clair_few_vuln.json") + parser = ClairParser() + findings = parser.get_findings(my_file_handle, None) + my_file_handle.close() + self.assertEqual(4, len(findings)) + + def test_many_findings_clair(self): + my_file_handle = open("unittests/scans/clair/clair_many_vul.json") parser = ClairParser() findings = parser.get_findings(my_file_handle, None) my_file_handle.close() @@ -23,3 +30,24 @@ def test_many_findings(self): self.assertEqual("CVE-2018-20839 - (systemd, 237-3ubuntu10.29)", finding.title) self.assertEqual(1, len(finding.unsaved_vulnerability_ids)) self.assertEqual("CVE-2018-20839", finding.unsaved_vulnerability_ids[0]) + + def test_parse_no_content_no_findings_clairklar(self): + my_file_handle = open("unittests/scans/clair/clairklar_empty.json") + parser = ClairParser() + findings = parser.get_findings(my_file_handle, None) + my_file_handle.close() + self.assertEqual(0, len(findings)) + + def test_high_findings_clairklar(self): + my_file_handle = open("unittests/scans/clair/clairklar_high.json") + parser = ClairParser() + findings = parser.get_findings(my_file_handle, None) + my_file_handle.close() + self.assertEqual(6, len(findings)) + + def test_mixed_findings_clairklar(self): + my_file_handle = open("unittests/scans/clair/clairklar_mixed.json") + parser = ClairParser() + findings = parser.get_findings(my_file_handle, None) + my_file_handle.close() + self.assertEqual(6, len(findings)) From 9414270f52fb8d765c4002c581f60bb7cb3da26a Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Thu, 18 Jan 2024 00:29:58 +0100 Subject: [PATCH 2/6] :bug: fix unittest --- unittests/test_import_reimport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unittests/test_import_reimport.py b/unittests/test_import_reimport.py index 2d0a7362b37..d765377b8f6 100644 --- a/unittests/test_import_reimport.py +++ b/unittests/test_import_reimport.py @@ -79,8 +79,8 @@ def __init__(self, *args, **kwargs): self.veracode_mitigated_findings = self.scans_path + 'veracode/mitigated_finding.xml' self.scan_type_veracode = 'Veracode Scan' - self.clair_few_findings = self.scans_path + 'clair/few_vuln.json' - self.clair_empty = self.scans_path + 'clair/empty.json' + self.clair_few_findings = self.scans_path + 'clair/clair_few_vuln.json' + self.clair_empty = self.scans_path + 'clair/clair_empty.json' self.scan_type_clair = 'Clair Scan' self.generic_filename_with_file = self.scans_path + "generic/test_with_image.json" From 80741fe6972659ba207d2486113ade0528c2b8c4 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Fri, 19 Jan 2024 17:42:49 +0100 Subject: [PATCH 3/6] merge conflict resolve --- docs/content/en/integrations/parsers/file/clair.md | 5 +---- docs/content/en/integrations/parsers/file/clair_klar.md | 6 ++++++ 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 docs/content/en/integrations/parsers/file/clair_klar.md diff --git a/docs/content/en/integrations/parsers/file/clair.md b/docs/content/en/integrations/parsers/file/clair.md index 6e7a0215ed5..e2e9bd6dc5a 100644 --- a/docs/content/en/integrations/parsers/file/clair.md +++ b/docs/content/en/integrations/parsers/file/clair.md @@ -2,7 +2,4 @@ title: "Clair Scan" toc_hide: true --- -Import JSON reports of Docker image vulnerabilities from clair or the clair klar client. - -### Sample Scan Data -Sample Clair and Clair Klar scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/clair). \ No newline at end of file +Import JSON reports of Docker image vulnerabilities. diff --git a/docs/content/en/integrations/parsers/file/clair_klar.md b/docs/content/en/integrations/parsers/file/clair_klar.md new file mode 100644 index 00000000000..4328a17bda8 --- /dev/null +++ b/docs/content/en/integrations/parsers/file/clair_klar.md @@ -0,0 +1,6 @@ +--- +title: "Clair Klar Scan" +toc_hide: true +--- +Import JSON reports of Docker image vulnerabilities from clair klar +client. \ No newline at end of file From 4bc72d38c172e399ef83d998a35b9b392c389acd Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 21 Jan 2024 09:27:35 +0100 Subject: [PATCH 4/6] updated release notes --- docs/content/en/getting_started/upgrading/2.31.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/content/en/getting_started/upgrading/2.31.md b/docs/content/en/getting_started/upgrading/2.31.md index c07494e8174..a252ccc57b3 100644 --- a/docs/content/en/getting_started/upgrading/2.31.md +++ b/docs/content/en/getting_started/upgrading/2.31.md @@ -2,6 +2,14 @@ title: 'Upgrading to DefectDojo Version 2.31.x' toc_hide: true weight: -20240102 -description: No special instructions. +description: breaking change --- -There are no special instructions for upgrading to 2.31.x. Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.31.0) for the contents of the release. + +To reduce the number of parsers, multiple parsers were merged together: +- OpenVAS XML and OpenVAS CSV were merged to OpenVAS Parser. There is a migration process built into the upgrade that will automatically convert exiting OpenVAS XML and OpenVAS CSV findings into OpenVAS Parser findings. +- Clair Scan and Clair Klar Scan were merged to Clair Scan. There is a migration process built into the upgrade that will automatically convert exiting Clair Klar Scan findings to Clair Scan findings. + +**Breaking Change** + - If there is any use of the above mentioned parsers in automated fashion via the import and reimport API endpoints, the `scan-type` parameter needs to be updated accordingly. + +For all other changes, check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.31.0) for the contents of the release. From 46471505a4883e43539d84d930d74edffcd6d545 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 21 Jan 2024 16:11:16 +0100 Subject: [PATCH 5/6] add db migrations --- dojo/db_migrations/0197_parser_merge.py | 92 +++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 dojo/db_migrations/0197_parser_merge.py diff --git a/dojo/db_migrations/0197_parser_merge.py b/dojo/db_migrations/0197_parser_merge.py new file mode 100644 index 00000000000..bb8c67421ae --- /dev/null +++ b/dojo/db_migrations/0197_parser_merge.py @@ -0,0 +1,92 @@ +from django.db import migrations +import logging + +logger = logging.getLogger(__name__) + + +OPENVAS_REFERENCES = ['OpenVAS CSV', 'OpenVAS XML'] +CLAIRKLAR_REFERENCES = ['Clair Klar Scan'] + + +# update the test type object as well as the scan type name +def update_openvas_test(test, openvas_test_type) -> None: + if test.test_type.name in OPENVAS_REFERENCES or test.scan_type in OPENVAS_REFERENCES: + test.test_type = openvas_test_type + test.scan_type = openvas_test_type.name + test.save() + +def update_clairklar_test(test, clairklar_test_type) -> None: + if test.test_type.name in CLAIRKLAR_REFERENCES or test.scan_type in CLAIRKLAR_REFERENCES: + test.test_type = clairklar_test_type + test.scan_type = clairklar_test_type.name + test.save() + +# Update the found_by field to remove OpenVAS CSV/ OpenVAS XML and add OpenVAS Parser +def update_openvas_finding(finding, openvas_test_type, openvascsv_test_type, openvasxml_test_type) -> None: + # Check if nessus is in found by list and remove + if openvascsv_test_type in finding.found_by.all(): + finding.found_by.remove(openvascsv_test_type.id) + # Check if nessus WAS is in found by list and remove + if openvasxml_test_type in finding.found_by.all(): + finding.found_by.remove(openvasxml_test_type.id) + # Check if tenable is already in list somehow before adding it + if openvas_test_type not in finding.found_by.all(): + finding.found_by.add(openvas_test_type.id) + finding.save() + +# Update the found_by field to remove Clair Klar Scan and add Clair Scan +def update_clairklar_finding(finding, clair_test_type, clairklar_test_type) -> None: + # Check if nessus is in found by list and remove + if clairklar_test_type in finding.found_by.all(): + finding.found_by.remove(clairklar_test_type.id) + # Check if tenable is already in list somehow before adding it + if clair_test_type not in finding.found_by.all(): + finding.found_by.add(clair_test_type.id) + finding.save() + +# Update all finding objects that came from OpenVAS CSV /XML reports +def migrate_openvas_parsers(apps, schema_editor): + finding_model = apps.get_model('dojo', 'Finding') + test_type_model = apps.get_model('dojo', 'Test_Type') + # Get or create OpenVAS Test Type and fetch the OpenVAS XML and OpenVAS CSV test types + openvas_test_type, _ = test_type_model.objects.get_or_create(name="OpenVAS Parser", active=True) + openvascsv_test_type = test_type_model.objects.filter(name="OpenVAS CSV").first() + openvasxml_test_type = test_type_model.objects.filter(name="OpenVAS XML").first() + # Get all the findings found by Nessus and Nessus WAS + findings = finding_model.objects.filter(test__scan_type__in=OPENVAS_REFERENCES) + logger.warning(f'We identified {findings.count()} OpenVAS CSV/ OpenVAS XML findings to migrate to OpenVAS Parser findings') + # Iterate over all findings and change + for finding in findings: + # Update the found by field + update_openvas_finding(finding, openvas_test_type, openvascsv_test_type, openvasxml_test_type) + # Update the test object + update_openvas_test(finding.test, openvas_test_type) + +# Update all finding objects that came from Clair Klar reports +def migrate_clairklar_parsers(apps, schema_editor): + finding_model = apps.get_model('dojo', 'Finding') + test_type_model = apps.get_model('dojo', 'Test_Type') + # Get or create Clair Scan Test Type and fetch the Clair Klar Scan test types + clair_test_type, _ = test_type_model.objects.get_or_create(name="Clair Scan", active=True) + clairklar_test_type = test_type_model.objects.filter(name="Clair Klar Scan").first() + # Get all the findings found by Clair Klar Scan + findings = finding_model.objects.filter(test__scan_type__in=OPENVAS_REFERENCES) + logger.warning(f'We identified {findings.count()} Clair Klar Scan findings to migrate to Clair Scan findings') + # Iterate over all findings and change + for finding in findings: + # Update the found by field + update_clairklar_finding(finding, clair_test_type, clairklar_test_type) + # Update the test object + update_clairklar_test(finding.test, clair_test_type) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0196_notifications_sla_breach_combined'), + ] + + operations = [ + migrations.RunPython(migrate_openvas_parsers), + migrations.RunPython(migrate_clairklar_parsers), + ] \ No newline at end of file From 091a4b3c7a66f5232d2f1e4715303693dc653aa6 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Mon, 22 Jan 2024 08:33:18 +0100 Subject: [PATCH 6/6] flake8 --- dojo/db_migrations/0197_parser_merge.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dojo/db_migrations/0197_parser_merge.py b/dojo/db_migrations/0197_parser_merge.py index bb8c67421ae..b00dac78a46 100644 --- a/dojo/db_migrations/0197_parser_merge.py +++ b/dojo/db_migrations/0197_parser_merge.py @@ -1,6 +1,7 @@ from django.db import migrations import logging + logger = logging.getLogger(__name__) @@ -15,12 +16,14 @@ def update_openvas_test(test, openvas_test_type) -> None: test.scan_type = openvas_test_type.name test.save() + def update_clairklar_test(test, clairklar_test_type) -> None: if test.test_type.name in CLAIRKLAR_REFERENCES or test.scan_type in CLAIRKLAR_REFERENCES: test.test_type = clairklar_test_type test.scan_type = clairklar_test_type.name test.save() + # Update the found_by field to remove OpenVAS CSV/ OpenVAS XML and add OpenVAS Parser def update_openvas_finding(finding, openvas_test_type, openvascsv_test_type, openvasxml_test_type) -> None: # Check if nessus is in found by list and remove @@ -34,6 +37,7 @@ def update_openvas_finding(finding, openvas_test_type, openvascsv_test_type, ope finding.found_by.add(openvas_test_type.id) finding.save() + # Update the found_by field to remove Clair Klar Scan and add Clair Scan def update_clairklar_finding(finding, clair_test_type, clairklar_test_type) -> None: # Check if nessus is in found by list and remove @@ -44,6 +48,7 @@ def update_clairklar_finding(finding, clair_test_type, clairklar_test_type) -> N finding.found_by.add(clair_test_type.id) finding.save() + # Update all finding objects that came from OpenVAS CSV /XML reports def migrate_openvas_parsers(apps, schema_editor): finding_model = apps.get_model('dojo', 'Finding') @@ -62,6 +67,7 @@ def migrate_openvas_parsers(apps, schema_editor): # Update the test object update_openvas_test(finding.test, openvas_test_type) + # Update all finding objects that came from Clair Klar reports def migrate_clairklar_parsers(apps, schema_editor): finding_model = apps.get_model('dojo', 'Finding') @@ -89,4 +95,4 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython(migrate_openvas_parsers), migrations.RunPython(migrate_clairklar_parsers), - ] \ No newline at end of file + ]