From 361d0f0185d4073ca260348abc3bd967b8d726ae Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 16:14:23 +0100 Subject: [PATCH 01/26] :sparkles: merge acunetix and acunetix360 --- dojo/tools/acunetix/parser.py | 311 +++++++++++++++++++++------------- 1 file changed, 192 insertions(+), 119 deletions(-) diff --git a/dojo/tools/acunetix/parser.py b/dojo/tools/acunetix/parser.py index 3227d20e188..f70d007e146 100644 --- a/dojo/tools/acunetix/parser.py +++ b/dojo/tools/acunetix/parser.py @@ -1,7 +1,8 @@ import hashlib import logging - +import json import dateutil +from dateutil import parser import html2text import hyperlink from cvss import parser as cvss_parser @@ -23,145 +24,217 @@ def get_label_for_scan_types(self, scan_type): def get_description_for_scan_types(self, scan_type): return "XML format" - def get_findings(self, xml_output, test): - root = parse(xml_output).getroot() - - dupes = dict() - for scan in root.findall("Scan"): - start_url = scan.findtext("StartURL") - if ":" not in start_url: - start_url = "//" + start_url - # get report date - if scan.findtext("StartTime") and "" != scan.findtext("StartTime"): - report_date = dateutil.parser.parse( - scan.findtext("StartTime") - ).date() - - for item in scan.findall("ReportItems/ReportItem"): + def get_findings(self, filename, test): + if str(filename.name).endswith('.xml'): + root = parse(filename).getroot() + dupes = dict() + for scan in root.findall("Scan"): + start_url = scan.findtext("StartURL") + if ":" not in start_url: + start_url = "//" + start_url + # get report date + if scan.findtext("StartTime") and "" != scan.findtext("StartTime"): + report_date = dateutil.parser.parse( + scan.findtext("StartTime") + ).date() + for item in scan.findall("ReportItems/ReportItem"): + finding = Finding( + test=test, + title=item.findtext("Name"), + severity=self.get_severity(item.findtext("Severity")), + description=html2text.html2text( + item.findtext("Description") + ).strip(), + false_p=self.get_false_positive( + item.findtext("IsFalsePositive") + ), + static_finding=True, + dynamic_finding=False, + nb_occurences=1, + ) + if item.findtext("Impact") and "" != item.findtext("Impact"): + finding.impact = item.findtext("Impact") + if item.findtext("Recommendation") and "" != item.findtext( + "Recommendation" + ): + finding.mitigation = item.findtext("Recommendation") + if report_date: + finding.date = report_date + if item.findtext("CWEList/CWE"): + finding.cwe = self.get_cwe_number( + item.findtext("CWEList/CWE") + ) + references = [] + for reference in item.findall("References/Reference"): + url = reference.findtext("URL") + db = reference.findtext("Database") or url + references.append(" * [{}]({})".format(db, url)) + if len(references) > 0: + finding.references = "\n".join(references) + if item.findtext("CVSS3/Descriptor"): + cvss_objects = cvss_parser.parse_cvss_from_text( + item.findtext("CVSS3/Descriptor") + ) + if len(cvss_objects) > 0: + finding.cvssv3 = cvss_objects[0].clean_vector() + # more description are in "Details" + if ( + item.findtext("Details") + and len(item.findtext("Details").strip()) > 0 + ): + finding.description += "\n\n**Details:**\n{}".format( + html2text.html2text(item.findtext("Details")) + ) + if ( + item.findtext("TechnicalDetails") + and len(item.findtext("TechnicalDetails").strip()) > 0 + ): + finding.description += ( + "\n\n**TechnicalDetails:**\n\n{}".format( + item.findtext("TechnicalDetails") + ) + ) + # add requests + finding.unsaved_req_resp = list() + if len(item.findall("TechnicalDetails/Request")): + finding.dynamic_finding = ( + True # if there is some requests it's dynamic + ) + finding.static_finding = ( + False # if there is some requests it's dynamic + ) + for request in item.findall("TechnicalDetails/Request"): + finding.unsaved_req_resp.append( + {"req": (request.text or ""), "resp": ""} + ) + # manage the endpoint + url = hyperlink.parse(start_url) + endpoint = Endpoint( + host=url.host, + port=url.port, + path=item.findtext("Affects"), + ) + if url.scheme is not None and "" != url.scheme: + endpoint.protocol = url.scheme + finding.unsaved_endpoints = [endpoint] + dupe_key = hashlib.sha256( + "|".join( + [ + finding.title, + str(finding.impact), + str(finding.mitigation), + ] + ).encode("utf-8") + ).hexdigest() + if dupe_key in dupes: + find = dupes[dupe_key] + # add details for the duplicate finding + if ( + item.findtext("Details") + and len(item.findtext("Details").strip()) > 0 + ): + find.description += ( + "\n-----\n\n**Details:**\n{}".format( + html2text.html2text(item.findtext("Details")) + ) + ) + find.unsaved_endpoints.extend(finding.unsaved_endpoints) + find.unsaved_req_resp.extend(finding.unsaved_req_resp) + find.nb_occurences += finding.nb_occurences + logger.debug( + "Duplicate finding : {defectdojo_title}".format( + defectdojo_title=finding.title + ) + ) + else: + dupes[dupe_key] = finding + elif str(filename.name).endswith('.json'): + data = json.load(filename) + dupes = dict() + scan_date = parser.parse(data["Generated"]) + text_maker = html2text.HTML2Text() + text_maker.body_width = 0 + + for item in data["Vulnerabilities"]: + title = item["Name"] + findingdetail = text_maker.handle(item.get("Description", "")) + if "Cwe" in item["Classification"]: + try: + cwe = int(item["Classification"]["Cwe"].split(",")[0]) + except BaseException: + cwe = None + else: + cwe = None + sev = item["Severity"] + if sev not in ["Info", "Low", "Medium", "High", "Critical"]: + sev = "Info" + mitigation = text_maker.handle(item.get("RemedialProcedure", "")) + references = text_maker.handle(item.get("RemedyReferences", "")) + if "LookupId" in item: + lookupId = item["LookupId"] + references = ( + f"https://online.acunetix360.com/issues/detail/{lookupId}\n" + + references + ) + url = item["Url"] + impact = text_maker.handle(item.get("Impact", "")) + dupe_key = title + request = item["HttpRequest"]["Content"] + if request is None or len(request) <= 0: + request = "Request Not Found" + response = item["HttpResponse"]["Content"] + if response is None or len(response) <= 0: + response = "Response Not Found" + finding = Finding( + title=title, test=test, - title=item.findtext("Name"), - severity=self.get_severity(item.findtext("Severity")), - description=html2text.html2text( - item.findtext("Description") - ).strip(), - false_p=self.get_false_positive( - item.findtext("IsFalsePositive") - ), + description=findingdetail, + severity=sev.title(), + mitigation=mitigation, + impact=impact, + date=scan_date, + references=references, + cwe=cwe, static_finding=True, - dynamic_finding=False, - nb_occurences=1, ) - if item.findtext("Impact") and "" != item.findtext("Impact"): - finding.impact = item.findtext("Impact") - - if item.findtext("Recommendation") and "" != item.findtext( - "Recommendation" + if ( + (item["Classification"] is not None) + and (item["Classification"]["Cvss"] is not None) + and (item["Classification"]["Cvss"]["Vector"] is not None) ): - finding.mitigation = item.findtext("Recommendation") - - if report_date: - finding.date = report_date - - if item.findtext("CWEList/CWE"): - finding.cwe = self.get_cwe_number( - item.findtext("CWEList/CWE") - ) - - references = [] - for reference in item.findall("References/Reference"): - url = reference.findtext("URL") - db = reference.findtext("Database") or url - references.append(" * [{}]({})".format(db, url)) - if len(references) > 0: - finding.references = "\n".join(references) - - if item.findtext("CVSS3/Descriptor"): cvss_objects = cvss_parser.parse_cvss_from_text( - item.findtext("CVSS3/Descriptor") + item["Classification"]["Cvss"]["Vector"] ) if len(cvss_objects) > 0: finding.cvssv3 = cvss_objects[0].clean_vector() - # more description are in "Details" - if ( - item.findtext("Details") - and len(item.findtext("Details").strip()) > 0 - ): - finding.description += "\n\n**Details:**\n{}".format( - html2text.html2text(item.findtext("Details")) - ) - if ( - item.findtext("TechnicalDetails") - and len(item.findtext("TechnicalDetails").strip()) > 0 - ): - finding.description += ( - "\n\n**TechnicalDetails:**\n\n{}".format( - item.findtext("TechnicalDetails") - ) - ) + if item["State"] is not None: + state = [x.strip() for x in item["State"].split(",")] + if "AcceptedRisk" in state: + finding.risk_accepted = True + finding.active = False + elif "FalsePositive" in state: + finding.false_p = True + finding.active = False - # add requests - finding.unsaved_req_resp = list() - if len(item.findall("TechnicalDetails/Request")): - finding.dynamic_finding = ( - True # if there is some requests it's dynamic - ) - finding.static_finding = ( - False # if there is some requests it's dynamic - ) - for request in item.findall("TechnicalDetails/Request"): - finding.unsaved_req_resp.append( - {"req": (request.text or ""), "resp": ""} - ) + finding.unsaved_req_resp = [{"req": request, "resp": response}] + finding.unsaved_endpoints = [Endpoint.from_uri(url)] - # manage the endpoint - url = hyperlink.parse(start_url) - endpoint = Endpoint( - host=url.host, - port=url.port, - path=item.findtext("Affects"), - ) - if url.scheme is not None and "" != url.scheme: - endpoint.protocol = url.scheme - finding.unsaved_endpoints = [endpoint] - - dupe_key = hashlib.sha256( - "|".join( - [ - finding.title, - str(finding.impact), - str(finding.mitigation), - ] - ).encode("utf-8") - ).hexdigest() + if item.get("FirstSeenDate"): + parseddate = parser.parse(item["FirstSeenDate"]) + finding.date = parseddate if dupe_key in dupes: find = dupes[dupe_key] - # add details for the duplicate finding - if ( - item.findtext("Details") - and len(item.findtext("Details").strip()) > 0 - ): - find.description += ( - "\n-----\n\n**Details:**\n{}".format( - html2text.html2text(item.findtext("Details")) - ) - ) - find.unsaved_endpoints.extend(finding.unsaved_endpoints) find.unsaved_req_resp.extend(finding.unsaved_req_resp) - find.nb_occurences += finding.nb_occurences - logger.debug( - "Duplicate finding : {defectdojo_title}".format( - defectdojo_title=finding.title - ) - ) + find.unsaved_endpoints.extend(finding.unsaved_endpoints) else: dupes[dupe_key] = finding - return list(dupes.values()) + def get_cwe_number(self, cwe): """ Returns cwe number. From 9304752a2f973282e4051ed30c65afb73c726959 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 16:15:34 +0100 Subject: [PATCH 02/26] move unittests together --- .../{acunetix360 => acunetix}/acunetix360_many_findings.json | 0 .../scans/{acunetix360 => acunetix}/acunetix360_multiple_cwe.json | 0 .../scans/{acunetix360 => acunetix}/acunetix360_one_finding.json | 0 .../acunetix360_one_finding_accepted_risk.json | 0 .../acunetix360_one_finding_false_positive.json | 0 .../scans/{acunetix360 => acunetix}/acunetix360_zero_finding.json | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename unittests/scans/{acunetix360 => acunetix}/acunetix360_many_findings.json (100%) rename unittests/scans/{acunetix360 => acunetix}/acunetix360_multiple_cwe.json (100%) rename unittests/scans/{acunetix360 => acunetix}/acunetix360_one_finding.json (100%) rename unittests/scans/{acunetix360 => acunetix}/acunetix360_one_finding_accepted_risk.json (100%) rename unittests/scans/{acunetix360 => acunetix}/acunetix360_one_finding_false_positive.json (100%) rename unittests/scans/{acunetix360 => acunetix}/acunetix360_zero_finding.json (100%) diff --git a/unittests/scans/acunetix360/acunetix360_many_findings.json b/unittests/scans/acunetix/acunetix360_many_findings.json similarity index 100% rename from unittests/scans/acunetix360/acunetix360_many_findings.json rename to unittests/scans/acunetix/acunetix360_many_findings.json diff --git a/unittests/scans/acunetix360/acunetix360_multiple_cwe.json b/unittests/scans/acunetix/acunetix360_multiple_cwe.json similarity index 100% rename from unittests/scans/acunetix360/acunetix360_multiple_cwe.json rename to unittests/scans/acunetix/acunetix360_multiple_cwe.json diff --git a/unittests/scans/acunetix360/acunetix360_one_finding.json b/unittests/scans/acunetix/acunetix360_one_finding.json similarity index 100% rename from unittests/scans/acunetix360/acunetix360_one_finding.json rename to unittests/scans/acunetix/acunetix360_one_finding.json diff --git a/unittests/scans/acunetix360/acunetix360_one_finding_accepted_risk.json b/unittests/scans/acunetix/acunetix360_one_finding_accepted_risk.json similarity index 100% rename from unittests/scans/acunetix360/acunetix360_one_finding_accepted_risk.json rename to unittests/scans/acunetix/acunetix360_one_finding_accepted_risk.json diff --git a/unittests/scans/acunetix360/acunetix360_one_finding_false_positive.json b/unittests/scans/acunetix/acunetix360_one_finding_false_positive.json similarity index 100% rename from unittests/scans/acunetix360/acunetix360_one_finding_false_positive.json rename to unittests/scans/acunetix/acunetix360_one_finding_false_positive.json diff --git a/unittests/scans/acunetix360/acunetix360_zero_finding.json b/unittests/scans/acunetix/acunetix360_zero_finding.json similarity index 100% rename from unittests/scans/acunetix360/acunetix360_zero_finding.json rename to unittests/scans/acunetix/acunetix360_zero_finding.json From 58bb4c30c7b2e372e0efde30f4ac4096ae669969 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 16:16:00 +0100 Subject: [PATCH 03/26] merge unittests --- unittests/tools/test_acunetix_parser.py | 121 ++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/unittests/tools/test_acunetix_parser.py b/unittests/tools/test_acunetix_parser.py index 0ee5be5dc35..8a4010a4778 100644 --- a/unittests/tools/test_acunetix_parser.py +++ b/unittests/tools/test_acunetix_parser.py @@ -201,3 +201,124 @@ def test_parse_file_with_example_com(self): self.assertIn('resp', req_resp) self.assertIsNotNone(req_resp['resp']) self.assertIsInstance(req_resp['resp'], str) + + def test_parse_file_with_one_finding(self): + testfile = open("unittests/scans/acunetix/acunetix360_one_finding.json") + parser = AcunetixParser() + findings = parser.get_findings(testfile, Test()) + self.assertEqual(1, len(findings)) + for finding in findings: + for endpoint in finding.unsaved_endpoints: + endpoint.clean() + with self.subTest(i=0): + finding = findings[0] + self.assertEqual("Medium", finding.severity) + self.assertEqual(16, finding.cwe) + self.assertIsNotNone(finding.description) + self.assertGreater(len(finding.description), 0) + self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N/E:H/RL:O/RC:C", finding.cvssv3) + self.assertEqual(1, len(finding.unsaved_endpoints)) + endpoint = finding.unsaved_endpoints[0] + self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") + self.assertEqual(finding.date, datetime(2021, 6, 16, 12, 30)) + self.assertIn("https://online.acunetix360.com/issues/detail/735f4503-e9eb-4b4c-4306-ad49020a4c4b", finding.references) + + def test_parse_file_with_one_finding_false_positive(self): + testfile = open("unittests/scans/acunetix/acunetix360_one_finding_false_positive.json") + parser = AcunetixParser() + findings = parser.get_findings(testfile, Test()) + self.assertEqual(1, len(findings)) + for finding in findings: + for endpoint in finding.unsaved_endpoints: + endpoint.clean() + with self.subTest(i=0): + finding = findings[0] + self.assertEqual("Medium", finding.severity) + self.assertEqual(16, finding.cwe) + self.assertIsNotNone(finding.description) + self.assertGreater(len(finding.description), 0) + self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N/E:H/RL:O/RC:C", finding.cvssv3) + self.assertEqual(1, len(finding.unsaved_endpoints)) + endpoint = finding.unsaved_endpoints[0] + self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") + self.assertTrue(finding.false_p) + + def test_parse_file_with_one_finding_risk_accepted(self): + testfile = open("unittests/scans/acunetix/acunetix360_one_finding_accepted_risk.json") + parser = AcunetixParser() + findings = parser.get_findings(testfile, Test()) + self.assertEqual(1, len(findings)) + for finding in findings: + for endpoint in finding.unsaved_endpoints: + endpoint.clean() + with self.subTest(i=0): + finding = findings[0] + self.assertEqual("Medium", finding.severity) + self.assertEqual(16, finding.cwe) + self.assertIsNotNone(finding.description) + self.assertGreater(len(finding.description), 0) + self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N/E:H/RL:O/RC:C", finding.cvssv3) + self.assertEqual(1, len(finding.unsaved_endpoints)) + endpoint = finding.unsaved_endpoints[0] + self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") + self.assertTrue(finding.risk_accepted) + + def test_parse_file_with_multiple_finding(self): + testfile = open("unittests/scans/acunetix/acunetix360_many_findings.json") + parser = AcunetixParser() + findings = parser.get_findings(testfile, Test()) + self.assertEqual(16, len(findings)) + for finding in findings: + for endpoint in finding.unsaved_endpoints: + endpoint.clean() + with self.subTest(i=0): + finding = findings[0] + self.assertEqual("Medium", finding.severity) + self.assertEqual(16, finding.cwe) + self.assertIsNotNone(finding.description) + self.assertGreater(len(finding.description), 0) + self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N/E:H/RL:O/RC:C", finding.cvssv3) + self.assertEqual(1, len(finding.unsaved_endpoints)) + endpoint = finding.unsaved_endpoints[0] + self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") + + with self.subTest(i=1): + finding = findings[1] + self.assertEqual("Critical", finding.severity) + self.assertEqual(89, finding.cwe) + self.assertIsNotNone(finding.description) + self.assertGreater(len(finding.description), 0) + self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", finding.cvssv3) + self.assertEqual(1, len(finding.unsaved_endpoints)) + endpoint = finding.unsaved_endpoints[0] + self.assertEqual(str(endpoint), "http://php.testsparker.com/artist.php?id=-1%20OR%2017-7=10") + + with self.subTest(i=2): + finding = findings[2] + self.assertEqual("Medium", finding.severity) + self.assertEqual(205, finding.cwe) + self.assertIsNotNone(finding.description) + self.assertGreater(len(finding.description), 0) + self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:U/C:N/I:L/A:N/E:H/RL:O/RC:C", finding.cvssv3) + self.assertEqual(1, len(finding.unsaved_endpoints)) + endpoint = finding.unsaved_endpoints[0] + self.assertEqual(str(endpoint), "http://php.testsparker.com") + + def test_parse_file_with_mulitple_cwe(self): + testfile = open("unittests/scans/acunetix/acunetix360_multiple_cwe.json") + parser = AcunetixParser() + findings = parser.get_findings(testfile, Test()) + self.assertEqual(1, len(findings)) + for finding in findings: + for endpoint in finding.unsaved_endpoints: + endpoint.clean() + with self.subTest(i=0): + finding = findings[0] + self.assertEqual("Medium", finding.severity) + self.assertEqual(16, finding.cwe) + self.assertIsNotNone(finding.description) + self.assertGreater(len(finding.description), 0) + self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N/E:H/RL:O/RC:C", finding.cvssv3) + self.assertEqual(1, len(finding.unsaved_endpoints)) + endpoint = finding.unsaved_endpoints[0] + self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") From 6a092a65893a3998c76fe9f389d9ea0cccd95f41 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 16:41:29 +0100 Subject: [PATCH 04/26] fix unittests --- unittests/tools/test_acunetix_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unittests/tools/test_acunetix_parser.py b/unittests/tools/test_acunetix_parser.py index 8a4010a4778..aba12c409d2 100644 --- a/unittests/tools/test_acunetix_parser.py +++ b/unittests/tools/test_acunetix_parser.py @@ -1,8 +1,8 @@ import datetime - from ..dojo_test_case import DojoTestCase from dojo.models import Test from dojo.tools.acunetix.parser import AcunetixParser +from datetime import datetime as date class TestAcunetixParser(DojoTestCase): @@ -220,7 +220,7 @@ def test_parse_file_with_one_finding(self): self.assertEqual(1, len(finding.unsaved_endpoints)) endpoint = finding.unsaved_endpoints[0] self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") - self.assertEqual(finding.date, datetime(2021, 6, 16, 12, 30)) + self.assertEqual(finding.date, date(2021, 6, 16, 12, 30)) self.assertIn("https://online.acunetix360.com/issues/detail/735f4503-e9eb-4b4c-4306-ad49020a4c4b", finding.references) def test_parse_file_with_one_finding_false_positive(self): From e0cad5550bccfb1dfb734a7dea08353578cd74b1 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 16:42:17 +0100 Subject: [PATCH 05/26] remove acunetix360 --- dojo/tools/acunetix360/__init__.py | 0 dojo/tools/acunetix360/parser.py | 104 ----------------- unittests/tools/test_acunetix360_parser.py | 128 --------------------- 3 files changed, 232 deletions(-) delete mode 100644 dojo/tools/acunetix360/__init__.py delete mode 100644 dojo/tools/acunetix360/parser.py delete mode 100644 unittests/tools/test_acunetix360_parser.py diff --git a/dojo/tools/acunetix360/__init__.py b/dojo/tools/acunetix360/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dojo/tools/acunetix360/parser.py b/dojo/tools/acunetix360/parser.py deleted file mode 100644 index 2639e4567f0..00000000000 --- a/dojo/tools/acunetix360/parser.py +++ /dev/null @@ -1,104 +0,0 @@ -import json -import html2text - -from cvss import parser as cvss_parser -from dateutil import parser -from dojo.models import Finding, Endpoint - - -class Acunetix360Parser(object): - def get_scan_types(self): - return ["Acunetix360 Scan"] - - def get_label_for_scan_types(self, scan_type): - return "Acunetix360 Scan" - - def get_description_for_scan_types(self, scan_type): - return "Acunetix360 JSON format." - - def get_findings(self, filename, test): - data = json.load(filename) - dupes = dict() - scan_date = parser.parse(data["Generated"]) - text_maker = html2text.HTML2Text() - text_maker.body_width = 0 - - for item in data["Vulnerabilities"]: - title = item["Name"] - findingdetail = text_maker.handle(item.get("Description", "")) - if "Cwe" in item["Classification"]: - try: - cwe = int(item["Classification"]["Cwe"].split(",")[0]) - except BaseException: - cwe = None - else: - cwe = None - sev = item["Severity"] - if sev not in ["Info", "Low", "Medium", "High", "Critical"]: - sev = "Info" - mitigation = text_maker.handle(item.get("RemedialProcedure", "")) - references = text_maker.handle(item.get("RemedyReferences", "")) - if "LookupId" in item: - lookupId = item["LookupId"] - references = ( - f"https://online.acunetix360.com/issues/detail/{lookupId}\n" - + references - ) - url = item["Url"] - impact = text_maker.handle(item.get("Impact", "")) - dupe_key = title - request = item["HttpRequest"]["Content"] - if request is None or len(request) <= 0: - request = "Request Not Found" - response = item["HttpResponse"]["Content"] - if response is None or len(response) <= 0: - response = "Response Not Found" - - finding = Finding( - title=title, - test=test, - description=findingdetail, - severity=sev.title(), - mitigation=mitigation, - impact=impact, - date=scan_date, - references=references, - cwe=cwe, - static_finding=True, - ) - - if ( - (item["Classification"] is not None) - and (item["Classification"]["Cvss"] is not None) - and (item["Classification"]["Cvss"]["Vector"] is not None) - ): - cvss_objects = cvss_parser.parse_cvss_from_text( - item["Classification"]["Cvss"]["Vector"] - ) - if len(cvss_objects) > 0: - finding.cvssv3 = cvss_objects[0].clean_vector() - - if item["State"] is not None: - state = [x.strip() for x in item["State"].split(",")] - if "AcceptedRisk" in state: - finding.risk_accepted = True - finding.active = False - elif "FalsePositive" in state: - finding.false_p = True - finding.active = False - - finding.unsaved_req_resp = [{"req": request, "resp": response}] - finding.unsaved_endpoints = [Endpoint.from_uri(url)] - - if item.get("FirstSeenDate"): - parseddate = parser.parse(item["FirstSeenDate"]) - finding.date = parseddate - - if dupe_key in dupes: - find = dupes[dupe_key] - find.unsaved_req_resp.extend(finding.unsaved_req_resp) - find.unsaved_endpoints.extend(finding.unsaved_endpoints) - else: - dupes[dupe_key] = finding - - return list(dupes.values()) diff --git a/unittests/tools/test_acunetix360_parser.py b/unittests/tools/test_acunetix360_parser.py deleted file mode 100644 index d491a1de2b1..00000000000 --- a/unittests/tools/test_acunetix360_parser.py +++ /dev/null @@ -1,128 +0,0 @@ -from ..dojo_test_case import DojoTestCase -from dojo.models import Test -from dojo.tools.acunetix360.parser import Acunetix360Parser -from datetime import datetime - - -class TestAcunetix360Parser(DojoTestCase): - - def test_parse_file_with_one_finding(self): - testfile = open("unittests/scans/acunetix360/acunetix360_one_finding.json") - parser = Acunetix360Parser() - findings = parser.get_findings(testfile, Test()) - self.assertEqual(1, len(findings)) - for finding in findings: - for endpoint in finding.unsaved_endpoints: - endpoint.clean() - with self.subTest(i=0): - finding = findings[0] - self.assertEqual("Medium", finding.severity) - self.assertEqual(16, finding.cwe) - self.assertIsNotNone(finding.description) - self.assertGreater(len(finding.description), 0) - self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N/E:H/RL:O/RC:C", finding.cvssv3) - self.assertEqual(1, len(finding.unsaved_endpoints)) - endpoint = finding.unsaved_endpoints[0] - self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") - self.assertEqual(finding.date, datetime(2021, 6, 16, 12, 30)) - self.assertIn("https://online.acunetix360.com/issues/detail/735f4503-e9eb-4b4c-4306-ad49020a4c4b", finding.references) - - def test_parse_file_with_one_finding_false_positive(self): - testfile = open("unittests/scans/acunetix360/acunetix360_one_finding_false_positive.json") - parser = Acunetix360Parser() - findings = parser.get_findings(testfile, Test()) - self.assertEqual(1, len(findings)) - for finding in findings: - for endpoint in finding.unsaved_endpoints: - endpoint.clean() - with self.subTest(i=0): - finding = findings[0] - self.assertEqual("Medium", finding.severity) - self.assertEqual(16, finding.cwe) - self.assertIsNotNone(finding.description) - self.assertGreater(len(finding.description), 0) - self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N/E:H/RL:O/RC:C", finding.cvssv3) - self.assertEqual(1, len(finding.unsaved_endpoints)) - endpoint = finding.unsaved_endpoints[0] - self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") - self.assertTrue(finding.false_p) - - def test_parse_file_with_one_finding_risk_accepted(self): - testfile = open("unittests/scans/acunetix360/acunetix360_one_finding_accepted_risk.json") - parser = Acunetix360Parser() - findings = parser.get_findings(testfile, Test()) - self.assertEqual(1, len(findings)) - for finding in findings: - for endpoint in finding.unsaved_endpoints: - endpoint.clean() - with self.subTest(i=0): - finding = findings[0] - self.assertEqual("Medium", finding.severity) - self.assertEqual(16, finding.cwe) - self.assertIsNotNone(finding.description) - self.assertGreater(len(finding.description), 0) - self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N/E:H/RL:O/RC:C", finding.cvssv3) - self.assertEqual(1, len(finding.unsaved_endpoints)) - endpoint = finding.unsaved_endpoints[0] - self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") - self.assertTrue(finding.risk_accepted) - - def test_parse_file_with_multiple_finding(self): - testfile = open("unittests/scans/acunetix360/acunetix360_many_findings.json") - parser = Acunetix360Parser() - findings = parser.get_findings(testfile, Test()) - self.assertEqual(16, len(findings)) - for finding in findings: - for endpoint in finding.unsaved_endpoints: - endpoint.clean() - with self.subTest(i=0): - finding = findings[0] - self.assertEqual("Medium", finding.severity) - self.assertEqual(16, finding.cwe) - self.assertIsNotNone(finding.description) - self.assertGreater(len(finding.description), 0) - self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N/E:H/RL:O/RC:C", finding.cvssv3) - self.assertEqual(1, len(finding.unsaved_endpoints)) - endpoint = finding.unsaved_endpoints[0] - self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") - - with self.subTest(i=1): - finding = findings[1] - self.assertEqual("Critical", finding.severity) - self.assertEqual(89, finding.cwe) - self.assertIsNotNone(finding.description) - self.assertGreater(len(finding.description), 0) - self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", finding.cvssv3) - self.assertEqual(1, len(finding.unsaved_endpoints)) - endpoint = finding.unsaved_endpoints[0] - self.assertEqual(str(endpoint), "http://php.testsparker.com/artist.php?id=-1%20OR%2017-7=10") - - with self.subTest(i=2): - finding = findings[2] - self.assertEqual("Medium", finding.severity) - self.assertEqual(205, finding.cwe) - self.assertIsNotNone(finding.description) - self.assertGreater(len(finding.description), 0) - self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:N/S:U/C:N/I:L/A:N/E:H/RL:O/RC:C", finding.cvssv3) - self.assertEqual(1, len(finding.unsaved_endpoints)) - endpoint = finding.unsaved_endpoints[0] - self.assertEqual(str(endpoint), "http://php.testsparker.com") - - def test_parse_file_with_mulitple_cwe(self): - testfile = open("unittests/scans/acunetix360/acunetix360_multiple_cwe.json") - parser = Acunetix360Parser() - findings = parser.get_findings(testfile, Test()) - self.assertEqual(1, len(findings)) - for finding in findings: - for endpoint in finding.unsaved_endpoints: - endpoint.clean() - with self.subTest(i=0): - finding = findings[0] - self.assertEqual("Medium", finding.severity) - self.assertEqual(16, finding.cwe) - self.assertIsNotNone(finding.description) - self.assertGreater(len(finding.description), 0) - self.assertEqual("CVSS:3.0/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:N/A:N/E:H/RL:O/RC:C", finding.cvssv3) - self.assertEqual(1, len(finding.unsaved_endpoints)) - endpoint = finding.unsaved_endpoints[0] - self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") From fb2c02d02915409a15a1814d06b1623128f6e560 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 16:43:28 +0100 Subject: [PATCH 06/26] update docs --- docs/content/en/integrations/parsers/file/acunetix.md | 2 +- docs/content/en/integrations/parsers/file/acunetix360.md | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 docs/content/en/integrations/parsers/file/acunetix360.md diff --git a/docs/content/en/integrations/parsers/file/acunetix.md b/docs/content/en/integrations/parsers/file/acunetix.md index 96a2c2005cc..97a2124e8ac 100644 --- a/docs/content/en/integrations/parsers/file/acunetix.md +++ b/docs/content/en/integrations/parsers/file/acunetix.md @@ -2,7 +2,7 @@ title: "Acunetix Scanner" toc_hide: true --- -XML format +This parser imports the Acunetix Scanner with xml output or Acunetix 360 Scanner with JSON output. ### Sample Scan Data Sample Acunetix Scanner scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/acunetix). \ No newline at end of file diff --git a/docs/content/en/integrations/parsers/file/acunetix360.md b/docs/content/en/integrations/parsers/file/acunetix360.md deleted file mode 100644 index 01b208bbeaa..00000000000 --- a/docs/content/en/integrations/parsers/file/acunetix360.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: "Acunetix 360 Scanner" -toc_hide: true ---- -Vulnerabilities List - JSON report - -### Sample Scan Data -Sample Acunetix 360 Scanner scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/acunetix360). \ No newline at end of file From 236cf414141e61b64bc9f2a6b0b70622b7b431f9 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 16:44:20 +0100 Subject: [PATCH 07/26] update get_description_for_scan_types --- dojo/tools/acunetix/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dojo/tools/acunetix/parser.py b/dojo/tools/acunetix/parser.py index f70d007e146..91a3d70f788 100644 --- a/dojo/tools/acunetix/parser.py +++ b/dojo/tools/acunetix/parser.py @@ -13,7 +13,7 @@ class AcunetixParser(object): - """Parser for Acunetix XML files.""" + """Parser for Acunetix XML files and Acunetix 360 JSON files.""" def get_scan_types(self): return ["Acunetix Scan"] @@ -22,7 +22,7 @@ def get_label_for_scan_types(self, scan_type): return "Acunetix Scanner" def get_description_for_scan_types(self, scan_type): - return "XML format" + return "Acunetix Scanner in XML format or Acunetix 360 Scanner in JSON format" def get_findings(self, filename, test): if str(filename.name).endswith('.xml'): From d175fbf4ce36dcdf8f86deb0b1af2ef232cb6ed4 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 16:46:17 +0100 Subject: [PATCH 08/26] flake8 --- dojo/tools/acunetix/parser.py | 1 - unittests/tools/test_acunetix_parser.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dojo/tools/acunetix/parser.py b/dojo/tools/acunetix/parser.py index 91a3d70f788..4f9833b4266 100644 --- a/dojo/tools/acunetix/parser.py +++ b/dojo/tools/acunetix/parser.py @@ -234,7 +234,6 @@ def get_findings(self, filename, test): dupes[dupe_key] = finding return list(dupes.values()) - def get_cwe_number(self, cwe): """ Returns cwe number. diff --git a/unittests/tools/test_acunetix_parser.py b/unittests/tools/test_acunetix_parser.py index aba12c409d2..eedf73b159f 100644 --- a/unittests/tools/test_acunetix_parser.py +++ b/unittests/tools/test_acunetix_parser.py @@ -202,7 +202,7 @@ def test_parse_file_with_example_com(self): self.assertIsNotNone(req_resp['resp']) self.assertIsInstance(req_resp['resp'], str) - def test_parse_file_with_one_finding(self): + def test_parse_file_with_one_finding_acunetix360(self): testfile = open("unittests/scans/acunetix/acunetix360_one_finding.json") parser = AcunetixParser() findings = parser.get_findings(testfile, Test()) @@ -263,7 +263,7 @@ def test_parse_file_with_one_finding_risk_accepted(self): self.assertEqual(str(endpoint), "http://php.testsparker.com/auth/login.php") self.assertTrue(finding.risk_accepted) - def test_parse_file_with_multiple_finding(self): + def test_parse_file_with_multiple_finding_acunetix360(self): testfile = open("unittests/scans/acunetix/acunetix360_many_findings.json") parser = AcunetixParser() findings = parser.get_findings(testfile, Test()) From ec0b8ad9f6712c6d00cffbe881fb088fd1edf861 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 19:01:32 +0100 Subject: [PATCH 09/26] :bug: fix unittests --- dojo/tools/acunetix/parser.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dojo/tools/acunetix/parser.py b/dojo/tools/acunetix/parser.py index 4f9833b4266..0d305a02c98 100644 --- a/dojo/tools/acunetix/parser.py +++ b/dojo/tools/acunetix/parser.py @@ -25,9 +25,9 @@ def get_description_for_scan_types(self, scan_type): return "Acunetix Scanner in XML format or Acunetix 360 Scanner in JSON format" def get_findings(self, filename, test): - if str(filename.name).endswith('.xml'): + dupes = dict() + if '.xml' in str(filename): root = parse(filename).getroot() - dupes = dict() for scan in root.findall("Scan"): start_url = scan.findtext("StartURL") if ":" not in start_url: @@ -148,7 +148,8 @@ def get_findings(self, filename, test): ) else: dupes[dupe_key] = finding - elif str(filename.name).endswith('.json'): + return list(dupes.values()) + elif '.json' in str(filename): data = json.load(filename) dupes = dict() scan_date = parser.parse(data["Generated"]) @@ -232,7 +233,7 @@ def get_findings(self, filename, test): find.unsaved_endpoints.extend(finding.unsaved_endpoints) else: dupes[dupe_key] = finding - return list(dupes.values()) + return list(dupes.values()) def get_cwe_number(self, cwe): """ From 35f9f641dc35186044e1a948d898a33d45685818 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 19:26:42 +0100 Subject: [PATCH 10/26] add db migrations --- dojo/db_migrations/0201_merge_acunetix.py | 55 +++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 dojo/db_migrations/0201_merge_acunetix.py diff --git a/dojo/db_migrations/0201_merge_acunetix.py b/dojo/db_migrations/0201_merge_acunetix.py new file mode 100644 index 00000000000..0e3eea65177 --- /dev/null +++ b/dojo/db_migrations/0201_merge_acunetix.py @@ -0,0 +1,55 @@ +from django.db import migrations +import logging + + +logger = logging.getLogger(__name__) + + +PARSER_REFERENCES = ['Acunetix360 Scan'] + + +def update_parser_test(test, parser_test_type) -> None: + if test.test_type.name in PARSER_REFERENCES or test.scan_type in PARSER_REFERENCES: + test.test_type = parser_test_type + test.scan_type = parser_test_type.name + test.save() + + +# Update the found_by field to remove Veracode SourceClear Scan and add Veracode Scan +def update_parser_finding(finding, newparser_test_type, parser_test_type) -> None: + # Check if nessus is in found by list and remove + if parser_test_type in finding.found_by.all(): + finding.found_by.remove(parser_test_type.id) + # Check if tenable is already in list somehow before adding it + if newparser_test_type not in finding.found_by.all(): + finding.found_by.add(newparser_test_type.id) + finding.save() + + +# Update all finding objects that came from Veracode SourceClear reports +def merge_parser(apps, schema_editor): + finding_model = apps.get_model('dojo', 'Finding') + test_type_model = apps.get_model('dojo', 'Test_Type') + # Get or create Veracode Scan Test Type and fetch the Veracode SourceClear Scan test types + newparser_test_type, _ = test_type_model.objects.get_or_create(name="Acunetix Scan", active=True) + parser_test_type = test_type_model.objects.filter(name="Acunetix360 Scan").first() + # Get all the findings found by Veracode SourceClear Scan + findings = finding_model.objects.filter(test__scan_type__in=PARSER_REFERENCES) + logger.warning(f'We identified {findings.count()} Acunetix360 Scan findings to migrate to Acunetix Scan findings') + # Iterate over all findings and change + for finding in findings: + # Update the found by field + update_parser_finding(finding, newparser_test_type, parser_test_type) + # Update the test object + update_parser_test(finding.test, newparser_test_type) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0200_finding_sla_expiration_date_product_async_updating_and_more'), + ] + + operations = [ + migrations.RunPython(merge_parser), + ] \ No newline at end of file From de42fe4641017f01566464410e517a976eb88ee8 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 11 Feb 2024 20:19:45 +0100 Subject: [PATCH 11/26] flake8,rufflinter --- dojo/db_migrations/0201_merge_acunetix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/db_migrations/0201_merge_acunetix.py b/dojo/db_migrations/0201_merge_acunetix.py index 0e3eea65177..74d09f2804c 100644 --- a/dojo/db_migrations/0201_merge_acunetix.py +++ b/dojo/db_migrations/0201_merge_acunetix.py @@ -52,4 +52,4 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython(merge_parser), - ] \ No newline at end of file + ] From 1ea5c847de19562f55595f0dc3162100c00f535b Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Thu, 15 Feb 2024 08:39:01 +0100 Subject: [PATCH 12/26] resolve db migrations problem in advance --- .../{0201_merge_acunetix.py => 0203_merge_acunetix.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename dojo/db_migrations/{0201_merge_acunetix.py => 0203_merge_acunetix.py} (96%) diff --git a/dojo/db_migrations/0201_merge_acunetix.py b/dojo/db_migrations/0203_merge_acunetix.py similarity index 96% rename from dojo/db_migrations/0201_merge_acunetix.py rename to dojo/db_migrations/0203_merge_acunetix.py index 74d09f2804c..534ec18d85b 100644 --- a/dojo/db_migrations/0201_merge_acunetix.py +++ b/dojo/db_migrations/0203_merge_acunetix.py @@ -47,7 +47,7 @@ def merge_parser(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('dojo', '0200_finding_sla_expiration_date_product_async_updating_and_more'), + ('dojo', '0202_alter_dojo_group_social_provider'), ] operations = [ From e292d5485a9a6bc98701a2540a04b155c033d275 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Wed, 21 Feb 2024 08:24:10 +0100 Subject: [PATCH 13/26] fix db migrations according to latest dev --- .../{0203_merge_acunetix.py => 0204_merge_acunetix.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename dojo/db_migrations/{0203_merge_acunetix.py => 0204_merge_acunetix.py} (96%) diff --git a/dojo/db_migrations/0203_merge_acunetix.py b/dojo/db_migrations/0204_merge_acunetix.py similarity index 96% rename from dojo/db_migrations/0203_merge_acunetix.py rename to dojo/db_migrations/0204_merge_acunetix.py index 534ec18d85b..5810eff247d 100644 --- a/dojo/db_migrations/0203_merge_acunetix.py +++ b/dojo/db_migrations/0204_merge_acunetix.py @@ -47,7 +47,7 @@ def merge_parser(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('dojo', '0202_alter_dojo_group_social_provider'), + ('dojo', '0203_alter_finding_options_finding_epss_percentile_and_more'), ] operations = [ From 704d6d01f689a4ea5067c0b57a133854431e40c1 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Wed, 21 Feb 2024 20:16:36 +0100 Subject: [PATCH 14/26] :bug: fix, see PR 9606 --- dojo/db_migrations/0204_merge_acunetix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/db_migrations/0204_merge_acunetix.py b/dojo/db_migrations/0204_merge_acunetix.py index 5810eff247d..193bd284487 100644 --- a/dojo/db_migrations/0204_merge_acunetix.py +++ b/dojo/db_migrations/0204_merge_acunetix.py @@ -31,7 +31,7 @@ def merge_parser(apps, schema_editor): finding_model = apps.get_model('dojo', 'Finding') test_type_model = apps.get_model('dojo', 'Test_Type') # Get or create Veracode Scan Test Type and fetch the Veracode SourceClear Scan test types - newparser_test_type, _ = test_type_model.objects.get_or_create(name="Acunetix Scan", active=True) + newparser_test_type, _ = test_type_model.objects.get_or_create(name="Acunetix Scan", defaults={"active": True}) parser_test_type = test_type_model.objects.filter(name="Acunetix360 Scan").first() # Get all the findings found by Veracode SourceClear Scan findings = finding_model.objects.filter(test__scan_type__in=PARSER_REFERENCES) From edf3dfe2531ff0cc7c92ddf348dfe509c4d99cd2 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Fri, 1 Mar 2024 08:14:46 +0100 Subject: [PATCH 15/26] basic structure update --- dojo/tools/acunetix/json_parser.py | 6 ++++++ dojo/tools/acunetix/parser.py | 2 ++ dojo/tools/acunetix/xml_parser.py | 6 ++++++ 3 files changed, 14 insertions(+) create mode 100644 dojo/tools/acunetix/json_parser.py create mode 100644 dojo/tools/acunetix/xml_parser.py diff --git a/dojo/tools/acunetix/json_parser.py b/dojo/tools/acunetix/json_parser.py new file mode 100644 index 00000000000..708c80ef754 --- /dev/null +++ b/dojo/tools/acunetix/json_parser.py @@ -0,0 +1,6 @@ + + +class AcunetixJSONParser(object): + """This parser is written for Acunetix JSON Findings.""" + def __init__(self) -> None: + pass \ No newline at end of file diff --git a/dojo/tools/acunetix/parser.py b/dojo/tools/acunetix/parser.py index 0d305a02c98..18d93cc7971 100644 --- a/dojo/tools/acunetix/parser.py +++ b/dojo/tools/acunetix/parser.py @@ -8,6 +8,8 @@ from cvss import parser as cvss_parser from defusedxml.ElementTree import parse from dojo.models import Endpoint, Finding +from dojo.tools.acunetix.json_parser import AcunetixJSONParser +from dojo.tools.acunetix.xml_parser import AcunetixXMLParser logger = logging.getLogger(__name__) diff --git a/dojo/tools/acunetix/xml_parser.py b/dojo/tools/acunetix/xml_parser.py new file mode 100644 index 00000000000..7156b1387cd --- /dev/null +++ b/dojo/tools/acunetix/xml_parser.py @@ -0,0 +1,6 @@ + + +class AcunetixXMLParser(object): + """This parser is written for Acunetix XML reports""" + def __init__(self) -> None: + pass \ No newline at end of file From 6385622f96f182f4d10e28124ac38d3250795022 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Fri, 1 Mar 2024 08:23:25 +0100 Subject: [PATCH 16/26] update acunetix xml --- ...on_parser.py => parse_acunetix360_json.py} | 0 dojo/tools/acunetix/parse_acunetix_xml.py | 135 ++++++++++++++++++ dojo/tools/acunetix/parser.py | 127 +--------------- dojo/tools/acunetix/xml_parser.py | 6 - 4 files changed, 138 insertions(+), 130 deletions(-) rename dojo/tools/acunetix/{json_parser.py => parse_acunetix360_json.py} (100%) create mode 100644 dojo/tools/acunetix/parse_acunetix_xml.py delete mode 100644 dojo/tools/acunetix/xml_parser.py diff --git a/dojo/tools/acunetix/json_parser.py b/dojo/tools/acunetix/parse_acunetix360_json.py similarity index 100% rename from dojo/tools/acunetix/json_parser.py rename to dojo/tools/acunetix/parse_acunetix360_json.py diff --git a/dojo/tools/acunetix/parse_acunetix_xml.py b/dojo/tools/acunetix/parse_acunetix_xml.py new file mode 100644 index 00000000000..eb06d50ff59 --- /dev/null +++ b/dojo/tools/acunetix/parse_acunetix_xml.py @@ -0,0 +1,135 @@ +import hashlib +import dateutil +import html2text +import hyperlink +from cvss import parser as cvss_parser +from defusedxml.ElementTree import parse +from dojo.models import Endpoint, Finding + + +class AcunetixXMLParser(object): + """This parser is written for Acunetix XML reports""" + def get_findings(self, filename, test): + dupes = dict() + root = parse(filename).getroot() + for scan in root.findall("Scan"): + start_url = scan.findtext("StartURL") + if ":" not in start_url: + start_url = "//" + start_url + # get report date + if scan.findtext("StartTime") and "" != scan.findtext("StartTime"): + report_date = dateutil.parser.parse( + scan.findtext("StartTime") + ).date() + for item in scan.findall("ReportItems/ReportItem"): + finding = Finding( + test=test, + title=item.findtext("Name"), + severity=self.get_severity(item.findtext("Severity")), + description=html2text.html2text( + item.findtext("Description") + ).strip(), + false_p=self.get_false_positive( + item.findtext("IsFalsePositive") + ), + static_finding=True, + dynamic_finding=False, + nb_occurences=1, + ) + if item.findtext("Impact") and "" != item.findtext("Impact"): + finding.impact = item.findtext("Impact") + if item.findtext("Recommendation") and "" != item.findtext( + "Recommendation" + ): + finding.mitigation = item.findtext("Recommendation") + if report_date: + finding.date = report_date + if item.findtext("CWEList/CWE"): + finding.cwe = self.get_cwe_number( + item.findtext("CWEList/CWE") + ) + references = [] + for reference in item.findall("References/Reference"): + url = reference.findtext("URL") + db = reference.findtext("Database") or url + references.append(" * [{}]({})".format(db, url)) + if len(references) > 0: + finding.references = "\n".join(references) + if item.findtext("CVSS3/Descriptor"): + cvss_objects = cvss_parser.parse_cvss_from_text( + item.findtext("CVSS3/Descriptor") + ) + if len(cvss_objects) > 0: + finding.cvssv3 = cvss_objects[0].clean_vector() + # more description are in "Details" + if ( + item.findtext("Details") + and len(item.findtext("Details").strip()) > 0 + ): + finding.description += "\n\n**Details:**\n{}".format( + html2text.html2text(item.findtext("Details")) + ) + if ( + item.findtext("TechnicalDetails") + and len(item.findtext("TechnicalDetails").strip()) > 0 + ): + finding.description += ( + "\n\n**TechnicalDetails:**\n\n{}".format( + item.findtext("TechnicalDetails") + ) + ) + # add requests + finding.unsaved_req_resp = list() + if len(item.findall("TechnicalDetails/Request")): + finding.dynamic_finding = ( + True # if there is some requests it's dynamic + ) + finding.static_finding = ( + False # if there is some requests it's dynamic + ) + for request in item.findall("TechnicalDetails/Request"): + finding.unsaved_req_resp.append( + {"req": (request.text or ""), "resp": ""} + ) + # manage the endpoint + url = hyperlink.parse(start_url) + endpoint = Endpoint( + host=url.host, + port=url.port, + path=item.findtext("Affects"), + ) + if url.scheme is not None and "" != url.scheme: + endpoint.protocol = url.scheme + finding.unsaved_endpoints = [endpoint] + dupe_key = hashlib.sha256( + "|".join( + [ + finding.title, + str(finding.impact), + str(finding.mitigation), + ] + ).encode("utf-8") + ).hexdigest() + if dupe_key in dupes: + find = dupes[dupe_key] + # add details for the duplicate finding + if ( + item.findtext("Details") + and len(item.findtext("Details").strip()) > 0 + ): + find.description += ( + "\n-----\n\n**Details:**\n{}".format( + html2text.html2text(item.findtext("Details")) + ) + ) + find.unsaved_endpoints.extend(finding.unsaved_endpoints) + find.unsaved_req_resp.extend(finding.unsaved_req_resp) + find.nb_occurences += finding.nb_occurences + logger.debug( + "Duplicate finding : {defectdojo_title}".format( + defectdojo_title=finding.title + ) + ) + else: + dupes[dupe_key] = finding + return list(dupes.values()) \ No newline at end of file diff --git a/dojo/tools/acunetix/parser.py b/dojo/tools/acunetix/parser.py index 18d93cc7971..9fb8d30db8a 100644 --- a/dojo/tools/acunetix/parser.py +++ b/dojo/tools/acunetix/parser.py @@ -8,8 +8,8 @@ from cvss import parser as cvss_parser from defusedxml.ElementTree import parse from dojo.models import Endpoint, Finding -from dojo.tools.acunetix.json_parser import AcunetixJSONParser -from dojo.tools.acunetix.xml_parser import AcunetixXMLParser +from dojo.tools.acunetix.parse_acunetix360_json import AcunetixJSONParser +from dojo.tools.acunetix.parse_acunetix_xml import AcunetixXMLParser logger = logging.getLogger(__name__) @@ -29,128 +29,7 @@ def get_description_for_scan_types(self, scan_type): def get_findings(self, filename, test): dupes = dict() if '.xml' in str(filename): - root = parse(filename).getroot() - for scan in root.findall("Scan"): - start_url = scan.findtext("StartURL") - if ":" not in start_url: - start_url = "//" + start_url - # get report date - if scan.findtext("StartTime") and "" != scan.findtext("StartTime"): - report_date = dateutil.parser.parse( - scan.findtext("StartTime") - ).date() - for item in scan.findall("ReportItems/ReportItem"): - finding = Finding( - test=test, - title=item.findtext("Name"), - severity=self.get_severity(item.findtext("Severity")), - description=html2text.html2text( - item.findtext("Description") - ).strip(), - false_p=self.get_false_positive( - item.findtext("IsFalsePositive") - ), - static_finding=True, - dynamic_finding=False, - nb_occurences=1, - ) - if item.findtext("Impact") and "" != item.findtext("Impact"): - finding.impact = item.findtext("Impact") - if item.findtext("Recommendation") and "" != item.findtext( - "Recommendation" - ): - finding.mitigation = item.findtext("Recommendation") - if report_date: - finding.date = report_date - if item.findtext("CWEList/CWE"): - finding.cwe = self.get_cwe_number( - item.findtext("CWEList/CWE") - ) - references = [] - for reference in item.findall("References/Reference"): - url = reference.findtext("URL") - db = reference.findtext("Database") or url - references.append(" * [{}]({})".format(db, url)) - if len(references) > 0: - finding.references = "\n".join(references) - if item.findtext("CVSS3/Descriptor"): - cvss_objects = cvss_parser.parse_cvss_from_text( - item.findtext("CVSS3/Descriptor") - ) - if len(cvss_objects) > 0: - finding.cvssv3 = cvss_objects[0].clean_vector() - # more description are in "Details" - if ( - item.findtext("Details") - and len(item.findtext("Details").strip()) > 0 - ): - finding.description += "\n\n**Details:**\n{}".format( - html2text.html2text(item.findtext("Details")) - ) - if ( - item.findtext("TechnicalDetails") - and len(item.findtext("TechnicalDetails").strip()) > 0 - ): - finding.description += ( - "\n\n**TechnicalDetails:**\n\n{}".format( - item.findtext("TechnicalDetails") - ) - ) - # add requests - finding.unsaved_req_resp = list() - if len(item.findall("TechnicalDetails/Request")): - finding.dynamic_finding = ( - True # if there is some requests it's dynamic - ) - finding.static_finding = ( - False # if there is some requests it's dynamic - ) - for request in item.findall("TechnicalDetails/Request"): - finding.unsaved_req_resp.append( - {"req": (request.text or ""), "resp": ""} - ) - # manage the endpoint - url = hyperlink.parse(start_url) - endpoint = Endpoint( - host=url.host, - port=url.port, - path=item.findtext("Affects"), - ) - if url.scheme is not None and "" != url.scheme: - endpoint.protocol = url.scheme - finding.unsaved_endpoints = [endpoint] - dupe_key = hashlib.sha256( - "|".join( - [ - finding.title, - str(finding.impact), - str(finding.mitigation), - ] - ).encode("utf-8") - ).hexdigest() - if dupe_key in dupes: - find = dupes[dupe_key] - # add details for the duplicate finding - if ( - item.findtext("Details") - and len(item.findtext("Details").strip()) > 0 - ): - find.description += ( - "\n-----\n\n**Details:**\n{}".format( - html2text.html2text(item.findtext("Details")) - ) - ) - find.unsaved_endpoints.extend(finding.unsaved_endpoints) - find.unsaved_req_resp.extend(finding.unsaved_req_resp) - find.nb_occurences += finding.nb_occurences - logger.debug( - "Duplicate finding : {defectdojo_title}".format( - defectdojo_title=finding.title - ) - ) - else: - dupes[dupe_key] = finding - return list(dupes.values()) + return AcunetixXMLParser().get_findings(filename, test) elif '.json' in str(filename): data = json.load(filename) dupes = dict() diff --git a/dojo/tools/acunetix/xml_parser.py b/dojo/tools/acunetix/xml_parser.py deleted file mode 100644 index 7156b1387cd..00000000000 --- a/dojo/tools/acunetix/xml_parser.py +++ /dev/null @@ -1,6 +0,0 @@ - - -class AcunetixXMLParser(object): - """This parser is written for Acunetix XML reports""" - def __init__(self) -> None: - pass \ No newline at end of file From 696a0e5508b458684d12908d7409122abc27d3c8 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Fri, 1 Mar 2024 08:25:45 +0100 Subject: [PATCH 17/26] update acunetix360 json --- dojo/tools/acunetix/parse_acunetix360_json.py | 132 ++++++++++++++++- dojo/tools/acunetix/parser.py | 134 +----------------- 2 files changed, 131 insertions(+), 135 deletions(-) diff --git a/dojo/tools/acunetix/parse_acunetix360_json.py b/dojo/tools/acunetix/parse_acunetix360_json.py index 708c80ef754..6c4129c4741 100644 --- a/dojo/tools/acunetix/parse_acunetix360_json.py +++ b/dojo/tools/acunetix/parse_acunetix360_json.py @@ -1,6 +1,134 @@ +import json +from dateutil import parser +import html2text +from cvss import parser as cvss_parser +from dojo.models import Endpoint, Finding class AcunetixJSONParser(object): """This parser is written for Acunetix JSON Findings.""" - def __init__(self) -> None: - pass \ No newline at end of file + def get_findings(self, filename, test): + dupes = dict() + data = json.load(filename) + dupes = dict() + scan_date = parser.parse(data["Generated"]) + text_maker = html2text.HTML2Text() + text_maker.body_width = 0 + + for item in data["Vulnerabilities"]: + title = item["Name"] + findingdetail = text_maker.handle(item.get("Description", "")) + if "Cwe" in item["Classification"]: + try: + cwe = int(item["Classification"]["Cwe"].split(",")[0]) + except BaseException: + cwe = None + else: + cwe = None + sev = item["Severity"] + if sev not in ["Info", "Low", "Medium", "High", "Critical"]: + sev = "Info" + mitigation = text_maker.handle(item.get("RemedialProcedure", "")) + references = text_maker.handle(item.get("RemedyReferences", "")) + if "LookupId" in item: + lookupId = item["LookupId"] + references = ( + f"https://online.acunetix360.com/issues/detail/{lookupId}\n" + + references + ) + url = item["Url"] + impact = text_maker.handle(item.get("Impact", "")) + dupe_key = title + request = item["HttpRequest"]["Content"] + if request is None or len(request) <= 0: + request = "Request Not Found" + response = item["HttpResponse"]["Content"] + if response is None or len(response) <= 0: + response = "Response Not Found" + + finding = Finding( + title=title, + test=test, + description=findingdetail, + severity=sev.title(), + mitigation=mitigation, + impact=impact, + date=scan_date, + references=references, + cwe=cwe, + static_finding=True, + ) + + if ( + (item["Classification"] is not None) + and (item["Classification"]["Cvss"] is not None) + and (item["Classification"]["Cvss"]["Vector"] is not None) + ): + cvss_objects = cvss_parser.parse_cvss_from_text( + item["Classification"]["Cvss"]["Vector"] + ) + if len(cvss_objects) > 0: + finding.cvssv3 = cvss_objects[0].clean_vector() + + if item["State"] is not None: + state = [x.strip() for x in item["State"].split(",")] + if "AcceptedRisk" in state: + finding.risk_accepted = True + finding.active = False + elif "FalsePositive" in state: + finding.false_p = True + finding.active = False + + finding.unsaved_req_resp = [{"req": request, "resp": response}] + finding.unsaved_endpoints = [Endpoint.from_uri(url)] + + if item.get("FirstSeenDate"): + parseddate = parser.parse(item["FirstSeenDate"]) + finding.date = parseddate + + if dupe_key in dupes: + find = dupes[dupe_key] + find.unsaved_req_resp.extend(finding.unsaved_req_resp) + find.unsaved_endpoints.extend(finding.unsaved_endpoints) + else: + dupes[dupe_key] = finding + return list(dupes.values()) + + def get_cwe_number(self, cwe): + """ + Returns cwe number. + :param cwe: + :return: cwe number + """ + if cwe is None: + return None + else: + return int(cwe.split("-")[1]) + + def get_severity(self, severity): + """ + Returns Severity as per DefectDojo standards. + :param severity: + :return: + """ + if severity == "high": + return "High" + elif severity == "medium": + return "Medium" + elif severity == "low": + return "Low" + elif severity == "informational": + return "Info" + else: + return "Critical" + + def get_false_positive(self, false_p): + """ + Returns True, False for false positive as per DefectDojo standards. + :param false_p: + :return: + """ + if false_p: + return True + else: + return False diff --git a/dojo/tools/acunetix/parser.py b/dojo/tools/acunetix/parser.py index 9fb8d30db8a..9347a20bb3c 100644 --- a/dojo/tools/acunetix/parser.py +++ b/dojo/tools/acunetix/parser.py @@ -1,13 +1,4 @@ -import hashlib import logging -import json -import dateutil -from dateutil import parser -import html2text -import hyperlink -from cvss import parser as cvss_parser -from defusedxml.ElementTree import parse -from dojo.models import Endpoint, Finding from dojo.tools.acunetix.parse_acunetix360_json import AcunetixJSONParser from dojo.tools.acunetix.parse_acunetix_xml import AcunetixXMLParser @@ -27,130 +18,7 @@ def get_description_for_scan_types(self, scan_type): return "Acunetix Scanner in XML format or Acunetix 360 Scanner in JSON format" def get_findings(self, filename, test): - dupes = dict() if '.xml' in str(filename): return AcunetixXMLParser().get_findings(filename, test) elif '.json' in str(filename): - data = json.load(filename) - dupes = dict() - scan_date = parser.parse(data["Generated"]) - text_maker = html2text.HTML2Text() - text_maker.body_width = 0 - - for item in data["Vulnerabilities"]: - title = item["Name"] - findingdetail = text_maker.handle(item.get("Description", "")) - if "Cwe" in item["Classification"]: - try: - cwe = int(item["Classification"]["Cwe"].split(",")[0]) - except BaseException: - cwe = None - else: - cwe = None - sev = item["Severity"] - if sev not in ["Info", "Low", "Medium", "High", "Critical"]: - sev = "Info" - mitigation = text_maker.handle(item.get("RemedialProcedure", "")) - references = text_maker.handle(item.get("RemedyReferences", "")) - if "LookupId" in item: - lookupId = item["LookupId"] - references = ( - f"https://online.acunetix360.com/issues/detail/{lookupId}\n" - + references - ) - url = item["Url"] - impact = text_maker.handle(item.get("Impact", "")) - dupe_key = title - request = item["HttpRequest"]["Content"] - if request is None or len(request) <= 0: - request = "Request Not Found" - response = item["HttpResponse"]["Content"] - if response is None or len(response) <= 0: - response = "Response Not Found" - - finding = Finding( - title=title, - test=test, - description=findingdetail, - severity=sev.title(), - mitigation=mitigation, - impact=impact, - date=scan_date, - references=references, - cwe=cwe, - static_finding=True, - ) - - if ( - (item["Classification"] is not None) - and (item["Classification"]["Cvss"] is not None) - and (item["Classification"]["Cvss"]["Vector"] is not None) - ): - cvss_objects = cvss_parser.parse_cvss_from_text( - item["Classification"]["Cvss"]["Vector"] - ) - if len(cvss_objects) > 0: - finding.cvssv3 = cvss_objects[0].clean_vector() - - if item["State"] is not None: - state = [x.strip() for x in item["State"].split(",")] - if "AcceptedRisk" in state: - finding.risk_accepted = True - finding.active = False - elif "FalsePositive" in state: - finding.false_p = True - finding.active = False - - finding.unsaved_req_resp = [{"req": request, "resp": response}] - finding.unsaved_endpoints = [Endpoint.from_uri(url)] - - if item.get("FirstSeenDate"): - parseddate = parser.parse(item["FirstSeenDate"]) - finding.date = parseddate - - if dupe_key in dupes: - find = dupes[dupe_key] - find.unsaved_req_resp.extend(finding.unsaved_req_resp) - find.unsaved_endpoints.extend(finding.unsaved_endpoints) - else: - dupes[dupe_key] = finding - return list(dupes.values()) - - def get_cwe_number(self, cwe): - """ - Returns cwe number. - :param cwe: - :return: cwe number - """ - if cwe is None: - return None - else: - return int(cwe.split("-")[1]) - - def get_severity(self, severity): - """ - Returns Severity as per DefectDojo standards. - :param severity: - :return: - """ - if severity == "high": - return "High" - elif severity == "medium": - return "Medium" - elif severity == "low": - return "Low" - elif severity == "informational": - return "Info" - else: - return "Critical" - - def get_false_positive(self, false_p): - """ - Returns True, False for false positive as per DefectDojo standards. - :param false_p: - :return: - """ - if false_p: - return True - else: - return False + return AcunetixJSONParser().get_findings(filename, test) From bd1e512eb0c8d6419d6a12f489ed7aa6238f4ac6 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Fri, 1 Mar 2024 08:32:57 +0100 Subject: [PATCH 18/26] :bug: fix --- dojo/tools/acunetix/parse_acunetix360_json.py | 39 +--------------- dojo/tools/acunetix/parse_acunetix_xml.py | 46 ++++++++++++++++++- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/dojo/tools/acunetix/parse_acunetix360_json.py b/dojo/tools/acunetix/parse_acunetix360_json.py index 6c4129c4741..b3702cbcc77 100644 --- a/dojo/tools/acunetix/parse_acunetix360_json.py +++ b/dojo/tools/acunetix/parse_acunetix360_json.py @@ -94,41 +94,4 @@ def get_findings(self, filename, test): dupes[dupe_key] = finding return list(dupes.values()) - def get_cwe_number(self, cwe): - """ - Returns cwe number. - :param cwe: - :return: cwe number - """ - if cwe is None: - return None - else: - return int(cwe.split("-")[1]) - - def get_severity(self, severity): - """ - Returns Severity as per DefectDojo standards. - :param severity: - :return: - """ - if severity == "high": - return "High" - elif severity == "medium": - return "Medium" - elif severity == "low": - return "Low" - elif severity == "informational": - return "Info" - else: - return "Critical" - - def get_false_positive(self, false_p): - """ - Returns True, False for false positive as per DefectDojo standards. - :param false_p: - :return: - """ - if false_p: - return True - else: - return False + \ No newline at end of file diff --git a/dojo/tools/acunetix/parse_acunetix_xml.py b/dojo/tools/acunetix/parse_acunetix_xml.py index eb06d50ff59..8ee49cc4535 100644 --- a/dojo/tools/acunetix/parse_acunetix_xml.py +++ b/dojo/tools/acunetix/parse_acunetix_xml.py @@ -1,11 +1,16 @@ import hashlib import dateutil import html2text +import logging + import hyperlink from cvss import parser as cvss_parser from defusedxml.ElementTree import parse from dojo.models import Endpoint, Finding +logger = logging.getLogger(__name__) + + class AcunetixXMLParser(object): """This parser is written for Acunetix XML reports""" @@ -132,4 +137,43 @@ def get_findings(self, filename, test): ) else: dupes[dupe_key] = finding - return list(dupes.values()) \ No newline at end of file + return list(dupes.values()) + + def get_cwe_number(self, cwe): + """ + Returns cwe number. + :param cwe: + :return: cwe number + """ + if cwe is None: + return None + else: + return int(cwe.split("-")[1]) + + def get_severity(self, severity): + """ + Returns Severity as per DefectDojo standards. + :param severity: + :return: + """ + if severity == "high": + return "High" + elif severity == "medium": + return "Medium" + elif severity == "low": + return "Low" + elif severity == "informational": + return "Info" + else: + return "Critical" + + def get_false_positive(self, false_p): + """ + Returns True, False for false positive as per DefectDojo standards. + :param false_p: + :return: + """ + if false_p: + return True + else: + return False From 56e7802300e70d1f98b0465e6ae1a62332d317b9 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Fri, 1 Mar 2024 08:34:43 +0100 Subject: [PATCH 19/26] flake8 --- dojo/tools/acunetix/parse_acunetix360_json.py | 8 -------- dojo/tools/acunetix/parse_acunetix_xml.py | 5 +---- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/dojo/tools/acunetix/parse_acunetix360_json.py b/dojo/tools/acunetix/parse_acunetix360_json.py index b3702cbcc77..f9fff0b109c 100644 --- a/dojo/tools/acunetix/parse_acunetix360_json.py +++ b/dojo/tools/acunetix/parse_acunetix360_json.py @@ -14,7 +14,6 @@ def get_findings(self, filename, test): scan_date = parser.parse(data["Generated"]) text_maker = html2text.HTML2Text() text_maker.body_width = 0 - for item in data["Vulnerabilities"]: title = item["Name"] findingdetail = text_maker.handle(item.get("Description", "")) @@ -45,7 +44,6 @@ def get_findings(self, filename, test): response = item["HttpResponse"]["Content"] if response is None or len(response) <= 0: response = "Response Not Found" - finding = Finding( title=title, test=test, @@ -58,7 +56,6 @@ def get_findings(self, filename, test): cwe=cwe, static_finding=True, ) - if ( (item["Classification"] is not None) and (item["Classification"]["Cvss"] is not None) @@ -78,14 +75,11 @@ def get_findings(self, filename, test): elif "FalsePositive" in state: finding.false_p = True finding.active = False - finding.unsaved_req_resp = [{"req": request, "resp": response}] finding.unsaved_endpoints = [Endpoint.from_uri(url)] - if item.get("FirstSeenDate"): parseddate = parser.parse(item["FirstSeenDate"]) finding.date = parseddate - if dupe_key in dupes: find = dupes[dupe_key] find.unsaved_req_resp.extend(finding.unsaved_req_resp) @@ -93,5 +87,3 @@ def get_findings(self, filename, test): else: dupes[dupe_key] = finding return list(dupes.values()) - - \ No newline at end of file diff --git a/dojo/tools/acunetix/parse_acunetix_xml.py b/dojo/tools/acunetix/parse_acunetix_xml.py index 8ee49cc4535..12ca4100a03 100644 --- a/dojo/tools/acunetix/parse_acunetix_xml.py +++ b/dojo/tools/acunetix/parse_acunetix_xml.py @@ -2,16 +2,13 @@ import dateutil import html2text import logging - import hyperlink from cvss import parser as cvss_parser from defusedxml.ElementTree import parse from dojo.models import Endpoint, Finding - logger = logging.getLogger(__name__) - class AcunetixXMLParser(object): """This parser is written for Acunetix XML reports""" def get_findings(self, filename, test): @@ -138,7 +135,7 @@ def get_findings(self, filename, test): else: dupes[dupe_key] = finding return list(dupes.values()) - + def get_cwe_number(self, cwe): """ Returns cwe number. From b45a8fa33b33d11e319594628353a1a37904aa9d Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Fri, 1 Mar 2024 08:48:09 +0100 Subject: [PATCH 20/26] update --- dojo/tools/acunetix/parser.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/dojo/tools/acunetix/parser.py b/dojo/tools/acunetix/parser.py index 9347a20bb3c..9d0ee771230 100644 --- a/dojo/tools/acunetix/parser.py +++ b/dojo/tools/acunetix/parser.py @@ -1,9 +1,6 @@ -import logging from dojo.tools.acunetix.parse_acunetix360_json import AcunetixJSONParser from dojo.tools.acunetix.parse_acunetix_xml import AcunetixXMLParser -logger = logging.getLogger(__name__) - class AcunetixParser(object): """Parser for Acunetix XML files and Acunetix 360 JSON files.""" From fe514eb8a14992e7bf36c9c37570ba10902cbff2 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Fri, 1 Mar 2024 09:26:06 +0100 Subject: [PATCH 21/26] :construction: db migration revert option --- dojo/db_migrations/0204_merge_acunetix.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/dojo/db_migrations/0204_merge_acunetix.py b/dojo/db_migrations/0204_merge_acunetix.py index 193bd284487..b56ccecefa4 100644 --- a/dojo/db_migrations/0204_merge_acunetix.py +++ b/dojo/db_migrations/0204_merge_acunetix.py @@ -15,7 +15,7 @@ def update_parser_test(test, parser_test_type) -> None: test.save() -# Update the found_by field to remove Veracode SourceClear Scan and add Veracode Scan +# Update the found_by field to remove Acunetix360 and add Acunetix def update_parser_finding(finding, newparser_test_type, parser_test_type) -> None: # Check if nessus is in found by list and remove if parser_test_type in finding.found_by.all(): @@ -26,14 +26,14 @@ def update_parser_finding(finding, newparser_test_type, parser_test_type) -> Non finding.save() -# Update all finding objects that came from Veracode SourceClear reports -def merge_parser(apps, schema_editor): +# Update all finding objects that came from Acunetix360 reports +def forward_merge_parser(apps, schema_editor): finding_model = apps.get_model('dojo', 'Finding') test_type_model = apps.get_model('dojo', 'Test_Type') - # Get or create Veracode Scan Test Type and fetch the Veracode SourceClear Scan test types + # Get or create Acunetix Scan Test Type and fetch the Acunetix360 Scan test types newparser_test_type, _ = test_type_model.objects.get_or_create(name="Acunetix Scan", defaults={"active": True}) parser_test_type = test_type_model.objects.filter(name="Acunetix360 Scan").first() - # Get all the findings found by Veracode SourceClear Scan + # Get all the findings found by Acunetix360 Scan findings = finding_model.objects.filter(test__scan_type__in=PARSER_REFERENCES) logger.warning(f'We identified {findings.count()} Acunetix360 Scan findings to migrate to Acunetix Scan findings') # Iterate over all findings and change @@ -43,6 +43,10 @@ def merge_parser(apps, schema_editor): # Update the test object update_parser_test(finding.test, newparser_test_type) +def reverse_uncouple_parser(apps, schema_editor): + finding_model = apps.get_model('dojo', 'Finding') + test_type_model = apps.get_model('dojo', 'Test_Type') + #TODO How can I distinguish now between original Acunetix360 findings and Acunetix findings if they were merged? class Migration(migrations.Migration): @@ -51,5 +55,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(merge_parser), + migrations.RunPython(forward_merge_parser, reverse_uncouple_parser), ] From 334d4ec967b3a999b422ae41d96f92bed0c5de12 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Mon, 4 Mar 2024 07:28:14 +0100 Subject: [PATCH 22/26] revert last commit --- dojo/db_migrations/0204_merge_acunetix.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dojo/db_migrations/0204_merge_acunetix.py b/dojo/db_migrations/0204_merge_acunetix.py index b56ccecefa4..f0487f99403 100644 --- a/dojo/db_migrations/0204_merge_acunetix.py +++ b/dojo/db_migrations/0204_merge_acunetix.py @@ -43,10 +43,6 @@ def forward_merge_parser(apps, schema_editor): # Update the test object update_parser_test(finding.test, newparser_test_type) -def reverse_uncouple_parser(apps, schema_editor): - finding_model = apps.get_model('dojo', 'Finding') - test_type_model = apps.get_model('dojo', 'Test_Type') - #TODO How can I distinguish now between original Acunetix360 findings and Acunetix findings if they were merged? class Migration(migrations.Migration): @@ -55,5 +51,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(forward_merge_parser, reverse_uncouple_parser), + migrations.RunPython(forward_merge_parser), ] From 0213c5e895edc9147d9e5527b6bbad6d11771734 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Wed, 6 Mar 2024 10:50:52 +0100 Subject: [PATCH 23/26] remove deduplication setting --- dojo/settings/settings.dist.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 0a01ae2245b..9689b8efa15 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -1201,7 +1201,6 @@ def saml2_attrib_map_format(dict): 'Symfony Security Check': ['title', 'vulnerability_ids'], 'DSOP Scan': ['vulnerability_ids'], 'Acunetix Scan': ['title', 'description'], - 'Acunetix360 Scan': ['title', 'description'], 'Terrascan Scan': ['vuln_id_from_tool', 'title', 'severity', 'file_path', 'line', 'component_name'], 'Trivy Operator Scan': ['title', 'severity', 'vulnerability_ids'], 'Trivy Scan': ['title', 'severity', 'vulnerability_ids', 'cwe', 'description'], @@ -1284,7 +1283,6 @@ def saml2_attrib_map_format(dict): 'Qualys Scan': True, 'DSOP Scan': True, 'Acunetix Scan': True, - 'Acunetix360 Scan': True, 'Trivy Operator Scan': True, 'Trivy Scan': True, 'SpotBugs Scan': False, @@ -1381,7 +1379,6 @@ def saml2_attrib_map_format(dict): 'Qualys Scan': DEDUPE_ALGO_HASH_CODE, 'PHP Symfony Security Check': DEDUPE_ALGO_HASH_CODE, 'Acunetix Scan': DEDUPE_ALGO_HASH_CODE, - 'Acunetix360 Scan': DEDUPE_ALGO_HASH_CODE, 'Clair Scan': DEDUPE_ALGO_HASH_CODE, 'Clair Klar Scan': DEDUPE_ALGO_HASH_CODE, # 'Qualys Webapp Scan': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL, # Must also uncomment qualys webapp line in hashcode fields per scanner From 6927566c5a97cb6c15dbb9bdfafcd9193ed2f2ae Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Wed, 6 Mar 2024 15:15:30 +0100 Subject: [PATCH 24/26] update db migrations --- .../{0204_merge_acunetix.py => 0205_merge_acunetix.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename dojo/db_migrations/{0204_merge_acunetix.py => 0205_merge_acunetix.py} (96%) diff --git a/dojo/db_migrations/0204_merge_acunetix.py b/dojo/db_migrations/0205_merge_acunetix.py similarity index 96% rename from dojo/db_migrations/0204_merge_acunetix.py rename to dojo/db_migrations/0205_merge_acunetix.py index f0487f99403..0ec3d08d7ae 100644 --- a/dojo/db_migrations/0204_merge_acunetix.py +++ b/dojo/db_migrations/0205_merge_acunetix.py @@ -47,7 +47,7 @@ def forward_merge_parser(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('dojo', '0203_alter_finding_options_finding_epss_percentile_and_more'), + ('dojo', '0204_jira_project_epic_issue_type_name'), ] operations = [ From fa5b17957c46db16c4b738199c8b812ebe14b02b Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Tue, 19 Mar 2024 11:32:29 +0100 Subject: [PATCH 25/26] adapt db migrations --- .../{0205_merge_acunetix.py => 0206_merge_acunetix.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename dojo/db_migrations/{0205_merge_acunetix.py => 0206_merge_acunetix.py} (97%) diff --git a/dojo/db_migrations/0205_merge_acunetix.py b/dojo/db_migrations/0206_merge_acunetix.py similarity index 97% rename from dojo/db_migrations/0205_merge_acunetix.py rename to dojo/db_migrations/0206_merge_acunetix.py index 0ec3d08d7ae..2d22e26f488 100644 --- a/dojo/db_migrations/0205_merge_acunetix.py +++ b/dojo/db_migrations/0206_merge_acunetix.py @@ -47,7 +47,7 @@ def forward_merge_parser(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('dojo', '0204_jira_project_epic_issue_type_name'), + ('dojo', '0205_jira_project_epic_issue_type_name'), ] operations = [ From 2faecb8a0c4085ad819d49117338d846ef8b5f87 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Mon, 25 Mar 2024 23:49:33 +0100 Subject: [PATCH 26/26] fix db migrations according to latest dev --- .../{0206_merge_acunetix.py => 0208_merge_acunetix.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename dojo/db_migrations/{0206_merge_acunetix.py => 0208_merge_acunetix.py} (97%) diff --git a/dojo/db_migrations/0206_merge_acunetix.py b/dojo/db_migrations/0208_merge_acunetix.py similarity index 97% rename from dojo/db_migrations/0206_merge_acunetix.py rename to dojo/db_migrations/0208_merge_acunetix.py index 2d22e26f488..601f4027144 100644 --- a/dojo/db_migrations/0206_merge_acunetix.py +++ b/dojo/db_migrations/0208_merge_acunetix.py @@ -47,7 +47,7 @@ def forward_merge_parser(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('dojo', '0205_jira_project_epic_issue_type_name'), + ('dojo', '0207_alter_sonarqube_issue_key'), ] operations = [