From 720f0781625169d822a4dcd65f7094a7261d0d5c Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 00:57:16 -0500 Subject: [PATCH 01/29] Update files in folder dojo/tools/mozilla_observatory with PEP8 standars. --- dojo/tools/mozilla_observatory/parser.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/dojo/tools/mozilla_observatory/parser.py b/dojo/tools/mozilla_observatory/parser.py index 0a268e5e525..72e6a6d6236 100644 --- a/dojo/tools/mozilla_observatory/parser.py +++ b/dojo/tools/mozilla_observatory/parser.py @@ -34,21 +34,25 @@ def get_findings(self, file, test): for key in nodes: node = nodes[key] - description = "\n".join([ - "**Score Description** : `" + node['score_description'] + "`", - "**Result** : `" + node['result'] + "`" - "**expectation** : " + str(node.get('expectation')) + "`", - ]) + description = "\n".join( + [ + "**Score Description** : `" + + node["score_description"] + + "`", + "**Result** : `" + node["result"] + "`" + "**expectation** : " + str(node.get("expectation")) + "`", + ] + ) finding = Finding( - title=node['score_description'], + title=node["score_description"], test=test, - active=not node['pass'], + active=not node["pass"], description=description, - severity=self.get_severity(int(node['score_modifier'])), + severity=self.get_severity(int(node["score_modifier"])), static_finding=False, dynamic_finding=True, - vuln_id_from_tool=node.get('name', key) + vuln_id_from_tool=node.get("name", key), ) findings.append(finding) From e6a2a93bc3670524a1151d7e2704e468e8a77a6b Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 00:58:12 -0500 Subject: [PATCH 02/29] Update files in folder dojo/tools/netsparker with PEP8 standars. --- dojo/tools/netsparker/parser.py | 50 +++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/dojo/tools/netsparker/parser.py b/dojo/tools/netsparker/parser.py index efc382ce37d..20d3b2e70b4 100644 --- a/dojo/tools/netsparker/parser.py +++ b/dojo/tools/netsparker/parser.py @@ -7,7 +7,6 @@ class NetsparkerParser(object): - def get_scan_types(self): return ["Netsparker Scan"] @@ -20,26 +19,27 @@ def get_description_for_scan_types(self, scan_type): def get_findings(self, filename, test): tree = filename.read() try: - data = json.loads(str(tree, 'utf-8-sig')) - except: + data = json.loads(str(tree, "utf-8-sig")) + except BaseException: data = json.loads(tree) dupes = dict() - scan_date = datetime.datetime.strptime(data["Generated"], "%d/%m/%Y %H:%M %p").date() + scan_date = datetime.datetime.strptime( + data["Generated"], "%d/%m/%Y %H:%M %p" + ).date() for item in data["Vulnerabilities"]: - title = item["Name"] findingdetail = html2text.html2text(item.get("Description", "")) if "Cwe" in item["Classification"]: try: - cwe = int(item["Classification"]["Cwe"].split(',')[0]) - except: + cwe = int(item["Classification"]["Cwe"].split(",")[0]) + except BaseException: cwe = None else: cwe = None sev = item["Severity"] - if sev not in ['Info', 'Low', 'Medium', 'High', 'Critical']: - sev = 'Info' + if sev not in ["Info", "Low", "Medium", "High", "Critical"]: + sev = "Info" mitigation = html2text.html2text(item.get("RemedialProcedure", "")) references = html2text.html2text(item.get("RemedyReferences", "")) url = item["Url"] @@ -48,16 +48,18 @@ def get_findings(self, filename, test): request = item["HttpRequest"]["Content"] response = item["HttpResponse"]["Content"] - finding = Finding(title=title, - test=test, - description=findingdetail, - severity=sev.title(), - mitigation=mitigation, - impact=impact, - date=scan_date, - references=references, - cwe=cwe, - static_finding=True) + finding = Finding( + title=title, + test=test, + description=findingdetail, + severity=sev.title(), + mitigation=mitigation, + impact=impact, + date=scan_date, + references=references, + cwe=cwe, + static_finding=True, + ) if item["State"].find("FalsePositive") != -1: finding.active = False @@ -69,8 +71,14 @@ def get_findings(self, filename, test): if item["State"].find("AcceptedRisk") != -1: finding.risk_accepted = True - if (item["Classification"] is not None) and (item["Classification"]["Cvss"] is not None) and (item["Classification"]["Cvss"]["Vector"] is not None): - cvss_objects = cvss_parser.parse_cvss_from_text(item["Classification"]["Cvss"]["Vector"]) + if ( + (item["Classification"] is not None) + and (item["Classification"]["Cvss"] is not None) + and (item["Classification"]["Cvss"]["Vector"] is not None) + ): + cvss_objects = cvss_parser.parse_cvss_from_text( + item["Classification"]["Cvss"]["Vector"] + ) if len(cvss_objects) > 0: finding.cvssv3 = cvss_objects[0].clean_vector() From cb55e769365f2d1982a1dc2d4c48da4cf0e4ce18 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 00:59:35 -0500 Subject: [PATCH 03/29] Update files in folder dojo/tools/neuvector with PEP8 standars. --- dojo/tools/neuvector/parser.py | 100 ++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 34 deletions(-) diff --git a/dojo/tools/neuvector/parser.py b/dojo/tools/neuvector/parser.py index 2607cfc1ef0..a3817a71ed8 100644 --- a/dojo/tools/neuvector/parser.py +++ b/dojo/tools/neuvector/parser.py @@ -5,9 +5,9 @@ logger = logging.getLogger(__name__) -NEUVECTOR_SCAN_NAME = 'NeuVector (REST)' -NEUVECTOR_IMAGE_SCAN_ENGAGEMENT_NAME = 'NV image scan' -NEUVECTOR_CONTAINER_SCAN_ENGAGEMENT_NAME = 'NV container scan' +NEUVECTOR_SCAN_NAME = "NeuVector (REST)" +NEUVECTOR_IMAGE_SCAN_ENGAGEMENT_NAME = "NV image scan" +NEUVECTOR_CONTAINER_SCAN_ENGAGEMENT_NAME = "NV container scan" class NeuVectorJsonParser(object): @@ -22,59 +22,92 @@ def parse_json(self, json_output): try: data = json_output.read() try: - tree = json.loads(str(data, 'utf-8')) - except: + tree = json.loads(str(data, "utf-8")) + except BaseException: tree = json.loads(data) - except: + except BaseException: raise ValueError("Invalid format") return tree def get_items(self, tree, test): items = {} - if 'report' in tree: - vulnerabilityTree = tree.get('report').get('vulnerabilities', []) + if "report" in tree: + vulnerabilityTree = tree.get("report").get("vulnerabilities", []) for node in vulnerabilityTree: item = get_item(node, test) - package_name = node.get('package_name') + package_name = node.get("package_name") if len(package_name) > 64: package_name = package_name[-64:] - unique_key = node.get('name') + str(package_name + str( - node.get('package_version')) + str(node.get('severity'))) + unique_key = node.get("name") + str( + package_name + + str(node.get("package_version")) + + str(node.get("severity")) + ) items[unique_key] = item return list(items.values()) def get_item(vulnerability, test): - severity = convert_severity(vulnerability.get('severity')) if 'severity' in vulnerability else "Info" - vector = vulnerability.get('vectors_v3') if 'vectors_v3' in vulnerability else "CVSSv3 vector not provided. " - fixed_version = vulnerability.get('fixed_version') if 'fixed_version' in vulnerability else "There seems to be no fix yet. Please check description field." - score_v3 = vulnerability.get('score_v3') if 'score_v3' in vulnerability else "No CVSSv3 score yet." - package_name = vulnerability.get('package_name') + severity = ( + convert_severity(vulnerability.get("severity")) + if "severity" in vulnerability + else "Info" + ) + vector = ( + vulnerability.get("vectors_v3") + if "vectors_v3" in vulnerability + else "CVSSv3 vector not provided. " + ) + fixed_version = ( + vulnerability.get("fixed_version") + if "fixed_version" in vulnerability + else "There seems to be no fix yet. Please check description field." + ) + score_v3 = ( + vulnerability.get("score_v3") + if "score_v3" in vulnerability + else "No CVSSv3 score yet." + ) + package_name = vulnerability.get("package_name") if len(package_name) > 64: package_name = package_name[-64:] - description = vulnerability.get('description') if 'description' in vulnerability else "" - link = vulnerability.get('link') if 'link' in vulnerability else "" + description = ( + vulnerability.get("description") + if "description" in vulnerability + else "" + ) + link = vulnerability.get("link") if "link" in vulnerability else "" # create the finding object finding = Finding( - title=vulnerability.get('name') + ": " + package_name + " - " + vulnerability.get('package_version'), + title=vulnerability.get("name") + + ": " + + package_name + + " - " + + vulnerability.get("package_version"), test=test, severity=severity, - description=description + "

Vulnerable Package: " + - package_name + "

Current Version: " + str( - vulnerability['package_version']) + "

", + description=description + + "

Vulnerable Package: " + + package_name + + "

Current Version: " + + str(vulnerability["package_version"]) + + "

", mitigation=fixed_version.title(), references=link, component_name=package_name, - component_version=vulnerability.get('package_version'), + component_version=vulnerability.get("package_version"), false_p=False, duplicate=False, out_of_scope=False, mitigated=None, - severity_justification="{} (CVSS v3 base score: {})\n".format(vector, score_v3), - impact=severity) - finding.unsaved_vulnerability_ids = [vulnerability.get('name')] + severity_justification="{} (CVSS v3 base score: {})\n".format( + vector, score_v3 + ), + impact=severity, + ) + finding.unsaved_vulnerability_ids = [vulnerability.get("name")] finding.description = finding.description.strip() return finding @@ -82,22 +115,21 @@ def get_item(vulnerability, test): # see neuvector/share/types.go def convert_severity(severity): - if severity.lower() == 'critical': + if severity.lower() == "critical": return "Critical" - elif severity.lower() == 'high': + elif severity.lower() == "high": return "High" - elif severity.lower() == 'medium': + elif severity.lower() == "medium": return "Medium" - elif severity.lower() == 'low': + elif severity.lower() == "low": return "Low" - elif severity == '': + elif severity == "": return "Info" else: return severity.title() class NeuVectorParser(object): - def get_scan_types(self): return [NEUVECTOR_SCAN_NAME] @@ -111,7 +143,7 @@ def get_findings(self, filename, test): if filename is None: return list() - if filename.name.lower().endswith('.json'): + if filename.name.lower().endswith(".json"): return NeuVectorJsonParser().parse(filename, test) else: - raise ValueError('Unknown File Format') + raise ValueError("Unknown File Format") From ae883239501e0f121d1246557eaaa44d6a8bd333 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:00:58 -0500 Subject: [PATCH 04/29] Update files in folder dojo/tools/neuvector_compliance with PEP8 standars. --- dojo/tools/neuvector_compliance/parser.py | 114 ++++++++++++---------- 1 file changed, 61 insertions(+), 53 deletions(-) diff --git a/dojo/tools/neuvector_compliance/parser.py b/dojo/tools/neuvector_compliance/parser.py index 16570caf3ab..f77f11603e9 100644 --- a/dojo/tools/neuvector_compliance/parser.py +++ b/dojo/tools/neuvector_compliance/parser.py @@ -4,7 +4,7 @@ from dojo.models import Finding -NEUVECTOR_SCAN_NAME = 'NeuVector (compliance)' +NEUVECTOR_SCAN_NAME = "NeuVector (compliance)" def parse(json_output, test): @@ -19,10 +19,10 @@ def parse_json(json_output): try: data = json_output.read() try: - tree = json.loads(str(data, 'utf-8')) - except: + tree = json.loads(str(data, "utf-8")) + except BaseException: tree = json.loads(data) - except: + except BaseException: raise ValueError("Invalid format") return tree @@ -36,98 +36,106 @@ def get_items(tree, test): # /v1/host/{id}/compliance or similar. thus, we need to support items in a # bit different leafs. testsTree = None - if 'report' in tree: - testsTree = tree.get('report').get('checks', []) + if "report" in tree: + testsTree = tree.get("report").get("checks", []) else: - testsTree = tree.get('items', []) + testsTree = tree.get("items", []) for node in testsTree: item = get_item(node, test) - unique_key = node.get('type') + node.get('category') + node.get('test_number') + node.get('description') - unique_key = hashlib.md5(unique_key.encode('utf-8')).hexdigest() + unique_key = ( + node.get("type") + + node.get("category") + + node.get("test_number") + + node.get("description") + ) + unique_key = hashlib.md5(unique_key.encode("utf-8")).hexdigest() items[unique_key] = item return list(items.values()) def get_item(node, test): - if 'test_number' not in node: + if "test_number" not in node: return None - if 'category' not in node: + if "category" not in node: return None - if 'description' not in node: + if "description" not in node: return None - if 'level' not in node: + if "level" not in node: return None - test_number = node.get('test_number') - test_description = node.get('description').rstrip() + test_number = node.get("test_number") + test_description = node.get("description").rstrip() - title = test_number + ' - ' + test_description + title = test_number + " - " + test_description - test_severity = node.get('level') + test_severity = node.get("level") severity = convert_severity(test_severity) - mitigation = node.get('remediation', '').rstrip() + mitigation = node.get("remediation", "").rstrip() - category = node.get('category') + category = node.get("category") - vuln_id_from_tool = category + '_' + test_number + vuln_id_from_tool = category + "_" + test_number - test_profile = node.get('profile', 'profile unknown') + test_profile = node.get("profile", "profile unknown") - full_description = '{} ({}), {}:\n'.format(test_number, category, test_profile) - full_description += '{}\n'.format(test_description) - full_description += 'Audit: {}\n'.format(test_severity) - if 'evidence' in node: - full_description += 'Evidence:\n{}\n'.format(node.get('evidence')) - if 'location' in node: - full_description += 'Location:\n{}\n'.format(node.get('location')) - full_description += 'Mitigation:\n{}\n'.format(mitigation) + full_description = "{} ({}), {}:\n".format( + test_number, category, test_profile + ) + full_description += "{}\n".format(test_description) + full_description += "Audit: {}\n".format(test_severity) + if "evidence" in node: + full_description += "Evidence:\n{}\n".format(node.get("evidence")) + if "location" in node: + full_description += "Location:\n{}\n".format(node.get("location")) + full_description += "Mitigation:\n{}\n".format(mitigation) - tags = node.get('tags', []) + tags = node.get("tags", []) if len(tags) > 0: - full_description += 'Tags:\n' + full_description += "Tags:\n" for t in tags: - full_description += '{}\n'.format(str(t).rstrip()) + full_description += "{}\n".format(str(t).rstrip()) - messages = node.get('message', []) + messages = node.get("message", []) if len(messages) > 0: - full_description += 'Messages:\n' + full_description += "Messages:\n" for m in messages: - full_description += '{}\n'.format(str(m).rstrip()) - - finding = Finding(title=title, - test=test, - description=full_description, - severity=severity, - mitigation=mitigation, - vuln_id_from_tool=vuln_id_from_tool, - static_finding=True, - dynamic_finding=False) + full_description += "{}\n".format(str(m).rstrip()) + + finding = Finding( + title=title, + test=test, + description=full_description, + severity=severity, + mitigation=mitigation, + vuln_id_from_tool=vuln_id_from_tool, + static_finding=True, + dynamic_finding=False, + ) return finding # see neuvector/share/clus_apis.go def convert_severity(severity): - if severity.lower() == 'high': + if severity.lower() == "high": return "High" - elif severity.lower() == 'warn': + elif severity.lower() == "warn": return "Medium" - elif severity.lower() == 'info': + elif severity.lower() == "info": return "Low" - elif severity.lower() == 'pass': + elif severity.lower() == "pass": return "Info" - elif severity.lower() == 'note': + elif severity.lower() == "note": return "Info" - elif severity.lower() == 'error': + elif severity.lower() == "error": return "Info" else: return severity.title() class NeuVectorComplianceParser(object): - def get_scan_types(self): return [NEUVECTOR_SCAN_NAME] @@ -141,7 +149,7 @@ def get_findings(self, filename, test): if filename is None: return list() - if filename.name.lower().endswith('.json'): + if filename.name.lower().endswith(".json"): return parse(filename, test) else: - raise ValueError('Unknown File Format') + raise ValueError("Unknown File Format") From 6d343fdfb74fbb3fa816bb17f0602559e4ecf55a Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:02:04 -0500 Subject: [PATCH 05/29] Update files in folder dojo/tools/nexpose with PEP8 standars. --- dojo/tools/nexpose/__init__.py | 2 +- dojo/tools/nexpose/parser.py | 311 +++++++++++++++++++-------------- 2 files changed, 180 insertions(+), 133 deletions(-) diff --git a/dojo/tools/nexpose/__init__.py b/dojo/tools/nexpose/__init__.py index 369f2551a3f..69e743a0066 100644 --- a/dojo/tools/nexpose/__init__.py +++ b/dojo/tools/nexpose/__init__.py @@ -1 +1 @@ -__author__ = 'jay7958' +__author__ = "jay7958" diff --git a/dojo/tools/nexpose/parser.py b/dojo/tools/nexpose/parser.py index cee5bb4ae9b..fc7a4344405 100644 --- a/dojo/tools/nexpose/parser.py +++ b/dojo/tools/nexpose/parser.py @@ -40,8 +40,7 @@ def parse_html_type(self, node): ret = "" tag = node.tag.lower() - if tag == 'containerblockelement': - + if tag == "containerblockelement": if len(list(node)) > 0: for child in list(node): ret += self.parse_html_type(child) @@ -52,19 +51,25 @@ def parse_html_type(self, node): ret += str(node.tail).strip() + "" else: ret += "" - if tag == 'listitem': + if tag == "listitem": if len(list(node)) > 0: for child in list(node): ret += self.parse_html_type(child) else: if node.text: ret += "
  • " + str(node.text).strip() + "
    • " - if tag == 'orderedlist': + if tag == "orderedlist": i = 1 for item in list(node): - ret += "
      " + str(i) + " " + self.parse_html_type(item) + "
    " + ret += ( + "
      " + + str(i) + + " " + + self.parse_html_type(item) + + "
    " + ) i += 1 - if tag == 'paragraph': + if tag == "paragraph": if len(list(node)) > 0: for child in list(node): ret += self.parse_html_type(child) @@ -75,12 +80,12 @@ def parse_html_type(self, node): ret += str(node.tail).strip() + "

    " else: ret += "

    " - if tag == 'unorderedlist': + if tag == "unorderedlist": for item in list(node): unorderedlist = self.parse_html_type(item) if unorderedlist not in ret: ret += "* " + unorderedlist - if tag == 'urllink': + if tag == "urllink": if node.text: ret += str(node.text).strip() + " " last = "" @@ -101,17 +106,24 @@ def parse_tests_type(self, node, vulnsDefinitions): """ vulns = list() - for tests in node.findall('tests'): - for test in tests.findall('test'): - if test.get('id') in vulnsDefinitions and ( - test.get('status') in ['vulnerable-exploited', 'vulnerable-version', 'vulnerable-potential']): - vuln = vulnsDefinitions[test.get('id').lower()] + for tests in node.findall("tests"): + for test in tests.findall("test"): + if test.get("id") in vulnsDefinitions and ( + test.get("status") + in [ + "vulnerable-exploited", + "vulnerable-version", + "vulnerable-potential", + ] + ): + vuln = vulnsDefinitions[test.get("id").lower()] for desc in list(test): - if 'pluginOutput' in vuln: - vuln['pluginOutput'] += "\n\n" + \ - self.parse_html_type(desc) + if "pluginOutput" in vuln: + vuln[ + "pluginOutput" + ] += "\n\n" + self.parse_html_type(desc) else: - vuln['pluginOutput'] = self.parse_html_type(desc) + vuln["pluginOutput"] = self.parse_html_type(desc) vulns.append(vuln) return vulns @@ -122,109 +134,137 @@ def get_vuln_definitions(self, tree): """ vulns = dict() url_index = 0 - for vulnsDef in tree.findall('VulnerabilityDefinitions'): - for vulnDef in vulnsDef.findall('vulnerability'): - vid = vulnDef.get('id').lower() - severity_chk = int(vulnDef.get('severity')) + for vulnsDef in tree.findall("VulnerabilityDefinitions"): + for vulnDef in vulnsDef.findall("vulnerability"): + vid = vulnDef.get("id").lower() + severity_chk = int(vulnDef.get("severity")) if severity_chk >= 9: - sev = 'Critical' + sev = "Critical" elif severity_chk >= 7: - sev = 'High' + sev = "High" elif severity_chk >= 4: - sev = 'Medium' + sev = "Medium" elif 0 < severity_chk < 4: - sev = 'Low' + sev = "Low" else: - sev = 'Info' + sev = "Info" vuln = { - 'desc': "", - 'name': vulnDef.get('title'), - 'vector': vulnDef.get('cvssVector'), # this is CVSS v2 - 'refs': dict(), - 'resolution': "", - 'severity': sev, - 'tags': list() + "desc": "", + "name": vulnDef.get("title"), + "vector": vulnDef.get("cvssVector"), # this is CVSS v2 + "refs": dict(), + "resolution": "", + "severity": sev, + "tags": list(), } for item in list(vulnDef): - if item.tag == 'description': + if item.tag == "description": for htmlType in list(item): - vuln['desc'] += self.parse_html_type(htmlType) + vuln["desc"] += self.parse_html_type(htmlType) - elif item.tag == 'exploits': + elif item.tag == "exploits": for exploit in list(item): - vuln['refs'][exploit.get('title')] = str(exploit.get('title')).strip() + ' ' + \ - str(exploit.get('link')).strip() + vuln["refs"][exploit.get("title")] = ( + str(exploit.get("title")).strip() + + " " + + str(exploit.get("link")).strip() + ) - elif item.tag == 'references': + elif item.tag == "references": for ref in list(item): - if 'URL' in ref.get('source'): - vuln['refs'][ref.get('source') + str(url_index)] = str(ref.text).strip() + if "URL" in ref.get("source"): + vuln["refs"][ + ref.get("source") + str(url_index) + ] = str(ref.text).strip() url_index += 1 else: - vuln['refs'][ref.get('source')] = str(ref.text).strip() + vuln["refs"][ref.get("source")] = str( + ref.text + ).strip() - elif item.tag == 'solution': + elif item.tag == "solution": for htmlType in list(item): - vuln['resolution'] += self.parse_html_type(htmlType) + vuln["resolution"] += self.parse_html_type( + htmlType + ) # there is currently no method to register tags in vulns - elif item.tag == 'tags': + elif item.tag == "tags": for tag in list(item): - vuln['tags'].append(tag.text.lower()) + vuln["tags"].append(tag.text.lower()) vulns[vid] = vuln return vulns def get_items(self, tree, vulns, test): hosts = list() - for nodes in tree.findall('nodes'): - for node in nodes.findall('node'): + for nodes in tree.findall("nodes"): + for node in nodes.findall("node"): host = dict() - host['name'] = node.get('address') - host['hostnames'] = set() - host['os'] = "" - host['services'] = list() - host['vulns'] = self.parse_tests_type(node, vulns) - - host['vulns'].append({ - 'name': 'Host Up', - 'desc': 'Host is up because it replied on ICMP request or some TCP/UDP port is up', - 'severity': 'Info', - }) - - for names in node.findall('names'): - for name in names.findall('name'): - host['hostnames'].add(name.text) - - for endpoints in node.findall('endpoints'): - for endpoint in endpoints.findall('endpoint'): + host["name"] = node.get("address") + host["hostnames"] = set() + host["os"] = "" + host["services"] = list() + host["vulns"] = self.parse_tests_type(node, vulns) + + host["vulns"].append( + { + "name": "Host Up", + "desc": "Host is up because it replied on ICMP request or some TCP/UDP port is up", + "severity": "Info", + } + ) + + for names in node.findall("names"): + for name in names.findall("name"): + host["hostnames"].add(name.text) + + for endpoints in node.findall("endpoints"): + for endpoint in endpoints.findall("endpoint"): svc = { - 'protocol': endpoint.get('protocol'), - 'port': int(endpoint.get('port')), - 'status': endpoint.get('status'), + "protocol": endpoint.get("protocol"), + "port": int(endpoint.get("port")), + "status": endpoint.get("status"), } - for services in endpoint.findall('services'): - for service in services.findall('service'): - svc['name'] = service.get('name', '').lower() - svc['vulns'] = self.parse_tests_type(service, vulns) - - for configs in service.findall('configurations'): - for config in configs.findall('config'): - if "banner" in config.get('name'): - svc['version'] = config.get('name') - - svc['vulns'].append({ - 'name': 'Open port {}/{}'.format(svc['protocol'].upper(), svc['port']), - 'desc': '{}/{} port is open with "{}" service'.format(svc['protocol'], - svc['port'], - service.get('name')), - 'severity': 'Info', - 'tags': [ - re.sub("[^A-Za-z0-9]+", "-", service.get('name').lower()).rstrip('-') - ] if service.get('name') != "" else [] - }) - - host['services'].append(svc) + for services in endpoint.findall("services"): + for service in services.findall("service"): + svc["name"] = service.get("name", "").lower() + svc["vulns"] = self.parse_tests_type( + service, vulns + ) + + for configs in service.findall( + "configurations" + ): + for config in configs.findall("config"): + if "banner" in config.get("name"): + svc["version"] = config.get("name") + + svc["vulns"].append( + { + "name": "Open port {}/{}".format( + svc["protocol"].upper(), + svc["port"], + ), + "desc": '{}/{} port is open with "{}" service'.format( + svc["protocol"], + svc["port"], + service.get("name"), + ), + "severity": "Info", + "tags": [ + re.sub( + "[^A-Za-z0-9]+", + "-", + service.get("name").lower(), + ).rstrip("-") + ] + if service.get("name") != "" + else [], + } + ) + + host["services"].append(svc) hosts.append(host) @@ -232,82 +272,89 @@ def get_items(self, tree, vulns, test): for host in hosts: # manage findings by node only - for vuln in host['vulns']: - dupe_key = vuln['severity'] + vuln['name'] + for vuln in host["vulns"]: + dupe_key = vuln["severity"] + vuln["name"] find = self.findings(dupe_key, dupes, test, vuln) - endpoint = Endpoint(host=host['name']) + endpoint = Endpoint(host=host["name"]) find.unsaved_endpoints.append(endpoint) - find.unsaved_tags = vuln.get('tags', []) + find.unsaved_tags = vuln.get("tags", []) # manage findings by service - for service in host['services']: - for vuln in service['vulns']: - dupe_key = vuln['severity'] + vuln['name'] + for service in host["services"]: + for vuln in service["vulns"]: + dupe_key = vuln["severity"] + vuln["name"] find = self.findings(dupe_key, dupes, test, vuln) endpoint = Endpoint( - host=host['name'], - port=service['port'], - protocol=service['name'] if service['name'] in SCHEME_PORT_MAP else service['protocol'], - fragment=service['protocol'].lower() if service['name'] == "dns" else None - # A little dirty hack but in case of DNS it is important to know if vulnerability is on TCP or UDP + host=host["name"], + port=service["port"], + protocol=service["name"] + if service["name"] in SCHEME_PORT_MAP + else service["protocol"], + fragment=service["protocol"].lower() + if service["name"] == "dns" + else None + # A little dirty hack but in case of DNS it is + # important to know if vulnerability is on TCP or UDP ) find.unsaved_endpoints.append(endpoint) - find.unsaved_tags = vuln.get('tags', []) + find.unsaved_tags = vuln.get("tags", []) return list(dupes.values()) @staticmethod def findings(dupe_key, dupes, test, vuln): - """ - - - """ + """ """ if dupe_key in dupes: find = dupes[dupe_key] - dupe_text = html2text.html2text(vuln.get('pluginOutput', '')) + dupe_text = html2text.html2text(vuln.get("pluginOutput", "")) if dupe_text not in find.description: find.description += "\n\n" + dupe_text else: - find = Finding(title=vuln['name'], - description=html2text.html2text( - vuln['desc'].strip()) + "\n\n" + html2text.html2text(vuln.get('pluginOutput', '').strip()), - severity=vuln['severity'], - mitigation=html2text.html2text(vuln.get('resolution')) if vuln.get('resolution') else None, - impact=vuln.get('vector') if vuln.get('vector') else None, - test=test, - false_p=False, - duplicate=False, - out_of_scope=False, - mitigated=None, - dynamic_finding=True) + find = Finding( + title=vuln["name"], + description=html2text.html2text(vuln["desc"].strip()) + + "\n\n" + + html2text.html2text(vuln.get("pluginOutput", "").strip()), + severity=vuln["severity"], + mitigation=html2text.html2text(vuln.get("resolution")) + if vuln.get("resolution") + else None, + impact=vuln.get("vector") if vuln.get("vector") else None, + test=test, + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated=None, + dynamic_finding=True, + ) # build references - refs = '' - for ref in vuln.get('refs', {}): - if ref.startswith('BID'): + refs = "" + for ref in vuln.get("refs", {}): + if ref.startswith("BID"): refs += f" * [{vuln['refs'][ref]}](https://www.securityfocus.com/bid/{vuln['refs'][ref]})" - elif ref.startswith('CA'): + elif ref.startswith("CA"): refs += f" * [{vuln['refs'][ref]}](https://www.cert.org/advisories/{vuln['refs'][ref]}.html)" - elif ref.startswith('CERT-VN'): + elif ref.startswith("CERT-VN"): refs += f" * [{vuln['refs'][ref]}](https://www.kb.cert.org/vuls/id/{vuln['refs'][ref]}.html)" - elif ref.startswith('CVE'): + elif ref.startswith("CVE"): refs += f" * [{vuln['refs'][ref]}](https://cve.mitre.org/cgi-bin/cvename.cgi?name={vuln['refs'][ref]})" - elif ref.startswith('DEBIAN'): + elif ref.startswith("DEBIAN"): refs += f" * [{vuln['refs'][ref]}](https://security-tracker.debian.org/tracker/{vuln['refs'][ref]})" - elif ref.startswith('XF'): + elif ref.startswith("XF"): refs += f" * [{vuln['refs'][ref]}](https://exchange.xforce.ibmcloud.com/vulnerabilities/{vuln['refs'][ref]})" - elif ref.startswith('URL'): + elif ref.startswith("URL"): refs += f" * URL: {vuln['refs'][ref]}" else: refs += f" * {ref}: {vuln['refs'][ref]}" refs += "\n" find.references = refs # update CVE - if "CVE" in vuln.get('refs', {}): - find.unsaved_vulnerability_ids = [vuln['refs']['CVE']] + if "CVE" in vuln.get("refs", {}): + find.unsaved_vulnerability_ids = [vuln["refs"]["CVE"]] find.unsaved_endpoints = list() dupes[dupe_key] = find return find From e6e7f09cb237c6540001f258eb8a22ba5de99c13 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:03:00 -0500 Subject: [PATCH 06/29] Update files in folder dojo/tools/nikto with PEP8 standars. --- dojo/tools/nikto/__init__.py | 2 +- dojo/tools/nikto/parser.py | 73 +++++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/dojo/tools/nikto/__init__.py b/dojo/tools/nikto/__init__.py index 369f2551a3f..69e743a0066 100644 --- a/dojo/tools/nikto/__init__.py +++ b/dojo/tools/nikto/__init__.py @@ -1 +1 @@ -__author__ = 'jay7958' +__author__ = "jay7958" diff --git a/dojo/tools/nikto/parser.py b/dojo/tools/nikto/parser.py index 37d969ab866..5092ba44d3d 100644 --- a/dojo/tools/nikto/parser.py +++ b/dojo/tools/nikto/parser.py @@ -1,4 +1,3 @@ - import hashlib import logging import re @@ -30,47 +29,53 @@ def get_label_for_scan_types(self, scan_type): return scan_type # no custom label for now def get_description_for_scan_types(self, scan_type): - return "XML output (old and new nxvmlversion=\"1.2\" type) or JSON output" + return ( + 'XML output (old and new nxvmlversion="1.2" type) or JSON output' + ) def get_findings(self, filename, test): - if filename.name.lower().endswith('.xml'): + if filename.name.lower().endswith(".xml"): return self.process_xml(filename, test) - elif filename.name.lower().endswith('.json'): + elif filename.name.lower().endswith(".json"): return self.process_json(filename, test) else: - raise ValueError('Unknown File Format') + raise ValueError("Unknown File Format") def process_json(self, file, test): data = json.load(file) dupes = dict() - host = data.get('host') - port = data.get('port') + host = data.get("host") + port = data.get("port") if port is not None: port = int(port) - for vulnerability in data.get('vulnerabilities', []): + for vulnerability in data.get("vulnerabilities", []): finding = Finding( - title=vulnerability.get('msg'), + title=vulnerability.get("msg"), severity="Info", # Nikto doesn't assign severity, default to Info - description="\n".join([ - f"**id:** `{vulnerability.get('id')}`", - f"**msg:** `{vulnerability.get('msg')}`", - f"**HTTP Method:** `{vulnerability.get('method')}`", - f"**OSVDB:** `{vulnerability.get('OSVDB')}`", - ]), - vuln_id_from_tool=vulnerability.get('id'), + description="\n".join( + [ + f"**id:** `{vulnerability.get('id')}`", + f"**msg:** `{vulnerability.get('msg')}`", + f"**HTTP Method:** `{vulnerability.get('method')}`", + f"**OSVDB:** `{vulnerability.get('OSVDB')}`", + ] + ), + vuln_id_from_tool=vulnerability.get("id"), nb_occurences=1, ) # manage if we have an ID from OSVDB - if "OSVDB" in vulnerability and "0" != vulnerability.get('OSVDB'): - finding.unique_id_from_tool = "OSVDB-" + vulnerability.get('OSVDB') + if "OSVDB" in vulnerability and "0" != vulnerability.get("OSVDB"): + finding.unique_id_from_tool = "OSVDB-" + vulnerability.get( + "OSVDB" + ) finding.description += "\n*This finding is marked as medium as there is a link to OSVDB*" finding.severity = "Medium" # build the endpoint endpoint = Endpoint( host=host, port=port, - path=vulnerability.get('url'), + path=vulnerability.get("url"), ) finding.unsaved_endpoints = [endpoint] @@ -80,8 +85,12 @@ def process_json(self, file, test): find = dupes[dupe_key] find.description += "\n-----\n" + finding.description find.unsaved_endpoints.append(endpoint) - find.unique_id_from_tool = None # as it is an aggregated finding we erase ids - find.vuln_id_from_tool = None # as it is an aggregated finding we erase ids + find.unique_id_from_tool = ( + None # as it is an aggregated finding we erase ids + ) + find.vuln_id_from_tool = ( + None # as it is an aggregated finding we erase ids + ) find.nb_occurences += 1 else: dupes[dupe_key] = finding @@ -93,41 +102,45 @@ def process_xml(self, file, test): tree = ET.parse(file) root = tree.getroot() - scan = root.find('scandetails') + scan = root.find("scandetails") if scan is not None: self.process_scandetail(scan, test, dupes) else: # New versions of Nikto have a new file type (nxvmlversion="1.2") which adds an additional niktoscan tag - # This find statement below is to support new file format while not breaking older Nikto scan files versions. - for scan in root.findall('./niktoscan/scandetails'): + # This find statement below is to support new file format while not + # breaking older Nikto scan files versions. + for scan in root.findall("./niktoscan/scandetails"): self.process_scandetail(scan, test, dupes) return list(dupes.values()) def process_scandetail(self, scan, test, dupes): - for item in scan.findall('item'): + for item in scan.findall("item"): # Title titleText = None description = item.findtext("description") # Cut the title down to the first sentence sentences = re.split( - r'(? 0: titleText = sentences[0][:900] else: titleText = description[:900] # Description - description = "\n".join([ + description = "\n".join( + [ f"**Host:** `{item.findtext('iplink')}`", f"**Description:** `{item.findtext('description')}`", f"**HTTP Method:** `{item.attrib.get('method')}`", - ]) + ] + ) # Manage severity the same way with JSON severity = "Info" # Nikto doesn't assign severity, default to Info - if item.get('osvdbid') is not None and "0" != item.get('osvdbid'): + if item.get("osvdbid") is not None and "0" != item.get("osvdbid"): severity = "Medium" finding = Finding( @@ -137,7 +150,7 @@ def process_scandetail(self, scan, test, dupes): severity=severity, dynamic_finding=True, static_finding=False, - vuln_id_from_tool=item.attrib.get('id'), + vuln_id_from_tool=item.attrib.get("id"), nb_occurences=1, ) From 572495741f2d864f7e9e7975ccc64743f0f8ea16 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:04:09 -0500 Subject: [PATCH 07/29] Update files in folder dojo/tools/nmap with PEP8 standars. --- dojo/tools/nmap/__init__.py | 2 +- dojo/tools/nmap/parser.py | 143 ++++++++++++++++++++++++------------ 2 files changed, 97 insertions(+), 48 deletions(-) diff --git a/dojo/tools/nmap/__init__.py b/dojo/tools/nmap/__init__.py index 43f000e0f3e..a7849c0c390 100644 --- a/dojo/tools/nmap/__init__.py +++ b/dojo/tools/nmap/__init__.py @@ -1 +1 @@ -__author__ = 'patriknordlen' +__author__ = "patriknordlen" diff --git a/dojo/tools/nmap/parser.py b/dojo/tools/nmap/parser.py index 5ac1f42290b..171795126c9 100755 --- a/dojo/tools/nmap/parser.py +++ b/dojo/tools/nmap/parser.py @@ -6,7 +6,6 @@ class NmapParser(object): - def get_scan_types(self): return ["Nmap Scan"] @@ -20,23 +19,29 @@ def get_findings(self, file, test): tree = parse(file) root = tree.getroot() dupes = dict() - if 'nmaprun' not in root.tag: + if "nmaprun" not in root.tag: raise ValueError("This doesn't seem to be a valid Nmap xml file.") report_date = None try: - report_date = datetime.datetime.fromtimestamp(int(root.attrib['start'])) + report_date = datetime.datetime.fromtimestamp( + int(root.attrib["start"]) + ) except ValueError: pass for host in root.findall("host"): host_info = "### Host\n\n" - ip = host.find("address[@addrtype='ipv4']").attrib['addr'] + ip = host.find("address[@addrtype='ipv4']").attrib["addr"] if ip is not None: host_info += "**IP Address:** %s\n" % ip - fqdn = host.find("hostnames/hostname[@type='PTR']").attrib['name'] if host.find("hostnames/hostname[@type='PTR']") is not None else None + fqdn = ( + host.find("hostnames/hostname[@type='PTR']").attrib["name"] + if host.find("hostnames/hostname[@type='PTR']") is not None + else None + ) if fqdn is not None: host_info += "**FQDN:** %s\n" % fqdn @@ -44,44 +49,70 @@ def get_findings(self, file, test): for os in host.iter("os"): for os_match in os.iter("osmatch"): - if 'name' in os_match.attrib: - host_info += "**Host OS:** %s\n" % os_match.attrib['name'] - if 'accuracy' in os_match.attrib: - host_info += "**Accuracy:** {0}%\n".format(os_match.attrib['accuracy']) + if "name" in os_match.attrib: + host_info += ( + "**Host OS:** %s\n" % os_match.attrib["name"] + ) + if "accuracy" in os_match.attrib: + host_info += "**Accuracy:** {0}%\n".format( + os_match.attrib["accuracy"] + ) host_info += "\n\n" for port_element in host.findall("ports/port"): - protocol = port_element.attrib['protocol'] - endpoint = Endpoint(host=fqdn if fqdn else ip, protocol=protocol) - if 'portid' in port_element.attrib and port_element.attrib['portid'].isdigit(): - endpoint.port = int(port_element.attrib['portid']) + protocol = port_element.attrib["protocol"] + endpoint = Endpoint( + host=fqdn if fqdn else ip, protocol=protocol + ) + if ( + "portid" in port_element.attrib + and port_element.attrib["portid"].isdigit() + ): + endpoint.port = int(port_element.attrib["portid"]) # filter on open ports - if 'open' != port_element.find("state").attrib.get('state'): + if "open" != port_element.find("state").attrib.get("state"): continue title = "Open port: %s/%s" % (endpoint.port, endpoint.protocol) description = host_info - description += "**Port/Protocol:** %s/%s\n" % (endpoint.port, endpoint.protocol) + description += "**Port/Protocol:** %s/%s\n" % ( + endpoint.port, + endpoint.protocol, + ) service_info = "\n\n" - if port_element.find('service') is not None: - if 'product' in port_element.find('service').attrib: - service_info += "**Product:** %s\n" % port_element.find('service').attrib['product'] - - if 'version' in port_element.find('service').attrib: - service_info += "**Version:** %s\n" % port_element.find('service').attrib['version'] - - if 'extrainfo' in port_element.find('service').attrib: - service_info += "**Extra Info:** %s\n" % port_element.find('service').attrib['extrainfo'] + if port_element.find("service") is not None: + if "product" in port_element.find("service").attrib: + service_info += ( + "**Product:** %s\n" + % port_element.find("service").attrib["product"] + ) + + if "version" in port_element.find("service").attrib: + service_info += ( + "**Version:** %s\n" + % port_element.find("service").attrib["version"] + ) + + if "extrainfo" in port_element.find("service").attrib: + service_info += ( + "**Extra Info:** %s\n" + % port_element.find("service").attrib["extrainfo"] + ) description += service_info description += "\n\n" - # manage some script like https://github.com/vulnersCom/nmap-vulners - for script_element in port_element.findall('script[@id="vulners"]'): - self.manage_vulner_script(test, dupes, script_element, endpoint, report_date) + # manage some script like + # https://github.com/vulnersCom/nmap-vulners + for script_element in port_element.findall( + 'script[@id="vulners"]' + ): + self.manage_vulner_script( + test, dupes, script_element, endpoint, report_date + ) severity = "Info" dupe_key = "nmap:" + str(endpoint.port) @@ -90,13 +121,14 @@ def get_findings(self, file, test): if description is not None: find.description += description else: - find = Finding(title=title, - test=test, - description=description, - severity=severity, - mitigation="N/A", - impact="No impact provided", - ) + find = Finding( + title=title, + test=test, + description=description, + severity=severity, + mitigation="N/A", + impact="No impact provided", + ) find.unsaved_endpoints = list() dupes[dupe_key] = find if report_date: @@ -124,37 +156,52 @@ def convert_cvss_score(self, raw_value): else: return "Critical" - def manage_vulner_script(self, test, dupes, script_element, endpoint, report_date=None): - for component_element in script_element.findall('table'): - component_cpe = CPE(component_element.attrib['key']) - for vuln in component_element.findall('table'): + def manage_vulner_script( + self, test, dupes, script_element, endpoint, report_date=None + ): + for component_element in script_element.findall("table"): + component_cpe = CPE(component_element.attrib["key"]) + for vuln in component_element.findall("table"): # convert elements in dict vuln_attributes = dict() - for elem in vuln.findall('elem'): - vuln_attributes[elem.attrib['key'].lower()] = elem.text + for elem in vuln.findall("elem"): + vuln_attributes[elem.attrib["key"].lower()] = elem.text - vuln_id = vuln_attributes['id'] + vuln_id = vuln_attributes["id"] description = "### Vulnerability\n\n" description += "**ID**: `" + str(vuln_id) + "`\n" description += "**CPE**: " + str(component_cpe) + "\n" for attribute in vuln_attributes: - description += "**" + attribute + "**: `" + vuln_attributes[attribute] + "`\n" - severity = self.convert_cvss_score(vuln_attributes['cvss']) + description += ( + "**" + + attribute + + "**: `" + + vuln_attributes[attribute] + + "`\n" + ) + severity = self.convert_cvss_score(vuln_attributes["cvss"]) finding = Finding( title=vuln_id, test=test, description=description, severity=severity, - component_name=component_cpe.get_product()[0] if len(component_cpe.get_product()) > 0 else '', - component_version=component_cpe.get_version()[0] if len(component_cpe.get_version()) > 0 else '', + component_name=component_cpe.get_product()[0] + if len(component_cpe.get_product()) > 0 + else "", + component_version=component_cpe.get_version()[0] + if len(component_cpe.get_version()) > 0 + else "", vuln_id_from_tool=vuln_id, nb_occurences=1, ) finding.unsaved_endpoints = [endpoint] # manage if CVE is in metadata - if "type" in vuln_attributes and "cve" == vuln_attributes["type"]: + if ( + "type" in vuln_attributes + and "cve" == vuln_attributes["type"] + ): finding.unsaved_vulnerability_ids = [vuln_attributes["id"]] if report_date: @@ -164,7 +211,9 @@ def manage_vulner_script(self, test, dupes, script_element, endpoint, report_dat if dupe_key in dupes: find = dupes[dupe_key] if description is not None: - find.description += "\n-----\n\n" + finding.description # fives '-' produces an horizontal line + find.description += ( + "\n-----\n\n" + finding.description + ) # fives '-' produces an horizontal line find.unsaved_endpoints.extend(finding.unsaved_endpoints) find.nb_occurences += finding.nb_occurences else: From 06e060656c2d9cbd1738bbe55b33022f9d26d66c Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:05:02 -0500 Subject: [PATCH 08/29] Update files in folder dojo/tools/npm_audit with PEP8 standars. --- dojo/tools/npm_audit/parser.py | 140 +++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 57 deletions(-) diff --git a/dojo/tools/npm_audit/parser.py b/dojo/tools/npm_audit/parser.py index 94aa5fae939..c3f97e49134 100644 --- a/dojo/tools/npm_audit/parser.py +++ b/dojo/tools/npm_audit/parser.py @@ -9,7 +9,6 @@ class NpmAuditParser(object): - def get_scan_types(self): return ["NPM Audit Scan"] @@ -29,22 +28,26 @@ def parse_json(self, json_output): try: data = json_output.read() try: - tree = json.loads(str(data, 'utf-8')) - except: + tree = json.loads(str(data, "utf-8")) + except BaseException: tree = json.loads(data) - except: + except BaseException: raise ValueError("Invalid format, unable to parse json.") - if tree.get('auditReportVersion'): - raise ValueError('npm7 with auditReportVersion 2 or higher not yet supported as it lacks the most important fields in the reports') + if tree.get("auditReportVersion"): + raise ValueError( + "npm7 with auditReportVersion 2 or higher not yet supported as it lacks the most important fields in the reports" + ) - if tree.get('error'): - error = tree.get('error') - code = error['code'] - summary = error['summary'] - raise ValueError('npm audit report contains errors: %s, %s', code, summary) + if tree.get("error"): + error = tree.get("error") + code = error["code"] + summary = error["summary"] + raise ValueError( + "npm audit report contains errors: %s, %s", code, summary + ) - subtree = tree.get('advisories') + subtree = tree.get("advisories") return subtree @@ -53,74 +56,97 @@ def get_items(self, tree, test): for key, node in tree.items(): item = get_item(node, test) - unique_key = str(node['id']) + str(node['module_name']) + unique_key = str(node["id"]) + str(node["module_name"]) items[unique_key] = item return list(items.values()) def censor_path_hashes(path): - """ https://github.com/npm/npm/issues/20739 for dependencies installed from git, npm audit replaces the name with a (random?) hash """ + """https://github.com/npm/npm/issues/20739 for dependencies installed from git, npm audit replaces the name with a (random?) hash""" """ this hash changes on every run of npm audit, so defect dojo might think it's a new finding every run """ """ we strip the hash and replace it with 'censored_by_npm_audit` """ if not path: return None - return re.sub('[a-f0-9]{64}', 'censored_by_npm_audit', path) + return re.sub("[a-f0-9]{64}", "censored_by_npm_audit", path) def get_item(item_node, test): - - if item_node['severity'] == 'low': - severity = 'Low' - elif item_node['severity'] == 'moderate': - severity = 'Medium' - elif item_node['severity'] == 'high': - severity = 'High' - elif item_node['severity'] == 'critical': - severity = 'Critical' + if item_node["severity"] == "low": + severity = "Low" + elif item_node["severity"] == "moderate": + severity = "Medium" + elif item_node["severity"] == "high": + severity = "High" + elif item_node["severity"] == "critical": + severity = "Critical" else: - severity = 'Info' + severity = "Info" - paths = '' + paths = "" component_version = None - for npm_finding in item_node['findings']: + for npm_finding in item_node["findings"]: # use first version as component_version - component_version = npm_finding['version'] if not component_version else component_version - paths += "\n - " + str(npm_finding['version']) + ":" + str(','.join(npm_finding['paths'][:25])) - if len(npm_finding['paths']) > 25: + component_version = ( + npm_finding["version"] + if not component_version + else component_version + ) + paths += ( + "\n - " + + str(npm_finding["version"]) + + ":" + + str(",".join(npm_finding["paths"][:25])) + ) + if len(npm_finding["paths"]) > 25: paths += "\n - ..... (list of paths truncated after 25 paths)" cwe = get_npm_cwe(item_node) - dojo_finding = Finding(title=item_node['title'] + " - " + "(" + item_node['module_name'] + ", " + item_node['vulnerable_versions'] + ")", - test=test, - severity=severity, - file_path=censor_path_hashes(item_node['findings'][0]['paths'][0]), - description=item_node['url'] + "\n" + - item_node['overview'] + "\n Vulnerable Module: " + - item_node['module_name'] + "\n Vulnerable Versions: " + - str(item_node['vulnerable_versions']) + "\n Patched Version: " + - str(item_node['patched_versions']) + "\n Vulnerable Paths: " + - str(paths) + "\n CWE: " + - str(item_node['cwe']) + "\n Access: " + - str(item_node['access']), - cwe=cwe, - mitigation=item_node['recommendation'], - references=item_node['url'], - component_name=item_node['module_name'], - component_version=component_version, - false_p=False, - duplicate=False, - out_of_scope=False, - mitigated=None, - impact="No impact provided", - static_finding=True, - dynamic_finding=False) - - if len(item_node['cves']) > 0: + dojo_finding = Finding( + title=item_node["title"] + + " - " + + "(" + + item_node["module_name"] + + ", " + + item_node["vulnerable_versions"] + + ")", + test=test, + severity=severity, + file_path=censor_path_hashes(item_node["findings"][0]["paths"][0]), + description=item_node["url"] + + "\n" + + item_node["overview"] + + "\n Vulnerable Module: " + + item_node["module_name"] + + "\n Vulnerable Versions: " + + str(item_node["vulnerable_versions"]) + + "\n Patched Version: " + + str(item_node["patched_versions"]) + + "\n Vulnerable Paths: " + + str(paths) + + "\n CWE: " + + str(item_node["cwe"]) + + "\n Access: " + + str(item_node["access"]), + cwe=cwe, + mitigation=item_node["recommendation"], + references=item_node["url"], + component_name=item_node["module_name"], + component_version=component_version, + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated=None, + impact="No impact provided", + static_finding=True, + dynamic_finding=False, + ) + + if len(item_node["cves"]) > 0: dojo_finding.unsaved_vulnerability_ids = list() - for vulnerability_id in item_node['cves']: + for vulnerability_id in item_node["cves"]: dojo_finding.unsaved_vulnerability_ids.append(vulnerability_id) return dojo_finding From 056ee69bc23bc15a1caf7a2f77a1080bb2211bde Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:06:21 -0500 Subject: [PATCH 09/29] Update files in folder dojo/tools/nsp with PEP8 standars. --- dojo/tools/nsp/parser.py | 66 ++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/dojo/tools/nsp/parser.py b/dojo/tools/nsp/parser.py index e628916c9e7..afec4b8ee93 100644 --- a/dojo/tools/nsp/parser.py +++ b/dojo/tools/nsp/parser.py @@ -4,7 +4,6 @@ class NspParser(object): - def get_scan_types(self): return ["Node Security Platform Scan"] @@ -25,10 +24,10 @@ def parse_json(self, json_output): try: data = json_output.read() try: - tree = json.loads(str(data, 'utf-8')) - except: + tree = json.loads(str(data, "utf-8")) + except BaseException: tree = json.loads(data) - except: + except BaseException: raise ValueError("Invalid format") return tree @@ -38,41 +37,56 @@ def get_items(self, tree, test): for node in tree: item = get_item(node, test) - unique_key = node['title'] + str(node['path']) + unique_key = node["title"] + str(node["path"]) items[unique_key] = item return list(items.values()) def get_item(item_node, test): - # Following the CVSS Scoring per https://nvd.nist.gov/vuln-metrics/cvss - if item_node['cvss_score'] <= 3.9: + if item_node["cvss_score"] <= 3.9: severity = "Low" - elif item_node['cvss_score'] > 4.0 and item_node['cvss_score'] <= 6.9: + elif item_node["cvss_score"] > 4.0 and item_node["cvss_score"] <= 6.9: severity = "Medium" - elif item_node['cvss_score'] > 7.0 and item_node['cvss_score'] <= 8.9: + elif item_node["cvss_score"] > 7.0 and item_node["cvss_score"] <= 8.9: severity = "High" else: severity = "Critical" - finding = Finding(title=item_node['title'] + " - " + "(" + item_node['module'] + ", " + item_node['version'] + ")", - test=test, - severity=severity, - description=item_node['overview'] + "\n Vulnerable Module: " + - item_node['module'] + "\n Vulnerable Versions: " + - str(item_node['vulnerable_versions']) + "\n Current Version: " + - str(item_node['version']) + "\n Patched Version: " + - str(item_node['patched_versions']) + "\n Vulnerable Path: " + " > ".join(item_node['path']) + "\n CVSS Score: " + - str(item_node['cvss_score']) + "\n CVSS Vector: " + - str(item_node['cvss_vector']), - mitigation=item_node['recommendation'], - references=item_node['advisory'], - false_p=False, - duplicate=False, - out_of_scope=False, - mitigated=None, - impact="No impact provided") + finding = Finding( + title=item_node["title"] + + " - " + + "(" + + item_node["module"] + + ", " + + item_node["version"] + + ")", + test=test, + severity=severity, + description=item_node["overview"] + + "\n Vulnerable Module: " + + item_node["module"] + + "\n Vulnerable Versions: " + + str(item_node["vulnerable_versions"]) + + "\n Current Version: " + + str(item_node["version"]) + + "\n Patched Version: " + + str(item_node["patched_versions"]) + + "\n Vulnerable Path: " + + " > ".join(item_node["path"]) + + "\n CVSS Score: " + + str(item_node["cvss_score"]) + + "\n CVSS Vector: " + + str(item_node["cvss_vector"]), + mitigation=item_node["recommendation"], + references=item_node["advisory"], + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated=None, + impact="No impact provided", + ) return finding From ffb6aa7d67e3787304b2b018443f7983ba33b108 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:07:23 -0500 Subject: [PATCH 10/29] Update files in folder dojo/tools/nuclei with PEP8 standars. --- dojo/tools/nuclei/parser.py | 144 +++++++++++++++++++++--------------- 1 file changed, 86 insertions(+), 58 deletions(-) diff --git a/dojo/tools/nuclei/parser.py b/dojo/tools/nuclei/parser.py index 21f6d07ffc7..782f0cf5786 100644 --- a/dojo/tools/nuclei/parser.py +++ b/dojo/tools/nuclei/parser.py @@ -14,7 +14,7 @@ class NucleiParser(object): A class that can be used to parse the nuclei (https://github.com/projectdiscovery/nuclei) JSON report file """ - DEFAULT_SEVERITY = 'Low' + DEFAULT_SEVERITY = "Low" def get_scan_types(self): return ["Nuclei Scan"] @@ -32,23 +32,26 @@ def get_findings(self, filename, test): dupes = {} for item in data: - logger.debug('Item %s.', str(item)) - template_id = item.get('templateID', item.get('template-id', '')) - info = item.get('info') - name = info.get('name') - severity = info.get('severity').title() + logger.debug("Item %s.", str(item)) + template_id = item.get("templateID", item.get("template-id", "")) + info = item.get("info") + name = info.get("name") + severity = info.get("severity").title() if severity not in Finding.SEVERITIES: - logger.debug('Unsupported severity value "%s", change to "%s"', - severity, self.DEFAULT_SEVERITY) + logger.debug( + 'Unsupported severity value "%s", change to "%s"', + severity, + self.DEFAULT_SEVERITY, + ) severity = self.DEFAULT_SEVERITY - item_type = item.get('type') + item_type = item.get("type") if item_type is None: - item_type = '' - matched = item.get('matched', item.get('matched-at', '')) - if '://' in matched: + item_type = "" + matched = item.get("matched", item.get("matched-at", "")) + if "://" in matched: endpoint = Endpoint.from_uri(matched) else: - endpoint = Endpoint.from_uri('//' + matched) + endpoint = Endpoint.from_uri("//" + matched) finding = Finding( title=f"{name}", @@ -57,72 +60,97 @@ def get_findings(self, filename, test): nb_occurences=1, vuln_id_from_tool=template_id, ) - if item.get('timestamp'): - finding.date = date_parser.parse(item.get('timestamp')) - if info.get('description'): - finding.description = info.get('description') - if item.get('extracted-results'): - finding.description += "\n**Results:**\n" + '\n'.join(item.get('extracted-results')) - if info.get('tags'): - finding.unsaved_tags = info.get('tags') - if info.get('reference'): - reference = info.get('reference') - if type(reference) is list: - finding.references = '\n'.join(info.get('reference')) + if item.get("timestamp"): + finding.date = date_parser.parse(item.get("timestamp")) + if info.get("description"): + finding.description = info.get("description") + if item.get("extracted-results"): + finding.description += "\n**Results:**\n" + "\n".join( + item.get("extracted-results") + ) + if info.get("tags"): + finding.unsaved_tags = info.get("tags") + if info.get("reference"): + reference = info.get("reference") + if isinstance(reference, list): + finding.references = "\n".join(info.get("reference")) else: - finding.references = info.get('reference') + finding.references = info.get("reference") finding.unsaved_endpoints.append(endpoint) - classification = info.get('classification') + classification = info.get("classification") if classification: - if 'cve-id' in classification and classification['cve-id']: - cve_ids = classification['cve-id'] - finding.unsaved_vulnerability_ids = list(map(lambda x: x.upper(), cve_ids)) - if ('cwe-id' in classification and classification['cwe-id'] - and len(classification['cwe-id']) > 0): - cwe = classification['cwe-id'][0] + if "cve-id" in classification and classification["cve-id"]: + cve_ids = classification["cve-id"] + finding.unsaved_vulnerability_ids = list( + map(lambda x: x.upper(), cve_ids) + ) + if ( + "cwe-id" in classification + and classification["cwe-id"] + and len(classification["cwe-id"]) > 0 + ): + cwe = classification["cwe-id"][0] finding.cwe = int(cwe[4:]) - if 'cvss-metrics' in classification and classification['cvss-metrics']: + if ( + "cvss-metrics" in classification + and classification["cvss-metrics"] + ): cvss_objects = cvss_parser.parse_cvss_from_text( - classification['cvss-metrics']) + classification["cvss-metrics"] + ) if len(cvss_objects) > 0: finding.cvssv3 = cvss_objects[0].clean_vector() - if 'cvss-score' in classification and classification['cvss-score']: - finding.cvssv3_score = classification['cvss-score'] + if ( + "cvss-score" in classification + and classification["cvss-score"] + ): + finding.cvssv3_score = classification["cvss-score"] - matcher = item.get('matcher-name', item.get('matcher_name')) + matcher = item.get("matcher-name", item.get("matcher_name")) if matcher: finding.component_name = matcher else: - matcher = '' - - if info.get('remediation'): - finding.mitigation = info.get('remediation') - - host = item.get('host', '') - - if item.get('curl-command'): - finding.steps_to_reproduce = 'curl command to reproduce the request:\n`' + \ - item.get('curl-command') + '`' - - if item.get('request'): - finding.unsaved_request = item.get('request') - if item.get('response'): - finding.unsaved_response = item.get('response') - - logger.debug('dupe keys %s, %s, %s, %s.', template_id, item_type, matcher, host) + matcher = "" + + if info.get("remediation"): + finding.mitigation = info.get("remediation") + + host = item.get("host", "") + + if item.get("curl-command"): + finding.steps_to_reproduce = ( + "curl command to reproduce the request:\n`" + + item.get("curl-command") + + "`" + ) + + if item.get("request"): + finding.unsaved_request = item.get("request") + if item.get("response"): + finding.unsaved_response = item.get("response") + + logger.debug( + "dupe keys %s, %s, %s, %s.", + template_id, + item_type, + matcher, + host, + ) dupe_key = hashlib.sha256( - (template_id + item_type + matcher + endpoint.host).encode('utf-8') + (template_id + item_type + matcher + endpoint.host).encode( + "utf-8" + ) ).hexdigest() if dupe_key in dupes: - logger.debug('dupe_key %s exists.', str(dupe_key)) + logger.debug("dupe_key %s exists.", str(dupe_key)) finding = dupes[dupe_key] if endpoint not in finding.unsaved_endpoints: finding.unsaved_endpoints.append(endpoint) - logger.debug('Appended endpoint %s', endpoint) + logger.debug("Appended endpoint %s", endpoint) finding.nb_occurences += 1 else: dupes[dupe_key] = finding From b761f80996b920996462e6a0820a793a0b574d68 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:08:37 -0500 Subject: [PATCH 11/29] Update files in folder dojo/tools/openscap with PEP8 standars. --- dojo/tools/openscap/parser.py | 90 ++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/dojo/tools/openscap/parser.py b/dojo/tools/openscap/parser.py index 784c335103f..9f3ba66132d 100644 --- a/dojo/tools/openscap/parser.py +++ b/dojo/tools/openscap/parser.py @@ -9,7 +9,6 @@ class OpenscapParser(object): - def get_scan_types(self): return ["Openscap Vulnerability Scan"] @@ -26,52 +25,75 @@ def get_findings(self, file, test): namespace = self.get_namespace(root) # check if xml file hash correct root or not. - if 'Benchmark' not in root.tag: - raise ValueError("This doesn't seem to be a valid Openscap vulnerability scan xml file.") - if 'http://checklists.nist.gov/xccdf/' not in namespace: - raise ValueError("This doesn't seem to be a valid Openscap vulnerability scan xml file.") + if "Benchmark" not in root.tag: + raise ValueError( + "This doesn't seem to be a valid Openscap vulnerability scan xml file." + ) + if "http://checklists.nist.gov/xccdf/" not in namespace: + raise ValueError( + "This doesn't seem to be a valid Openscap vulnerability scan xml file." + ) # read rules rules = {} - for rule in root.findall('.//{0}Rule'.format(namespace)): - rules[rule.attrib['id']] = { - "title": rule.findtext('./{0}title'.format(namespace)) + for rule in root.findall(".//{0}Rule".format(namespace)): + rules[rule.attrib["id"]] = { + "title": rule.findtext("./{0}title".format(namespace)) } # go to test result - test_result = tree.find('./{0}TestResult'.format(namespace)) + test_result = tree.find("./{0}TestResult".format(namespace)) ips = [] # append all target in a list. - for ip in test_result.findall('./{0}target'.format(namespace)): + for ip in test_result.findall("./{0}target".format(namespace)): ips.append(ip.text) - for ip in test_result.findall('./{0}target-address'.format(namespace)): + for ip in test_result.findall("./{0}target-address".format(namespace)): ips.append(ip.text) dupes = dict() - # run both rule, and rule-result in parallel so that we can get title for failed test from rule. - for rule_result in test_result.findall('./{0}rule-result'.format(namespace)): - result = rule_result.findtext('./{0}result'.format(namespace)) + # run both rule, and rule-result in parallel so that we can get title + # for failed test from rule. + for rule_result in test_result.findall( + "./{0}rule-result".format(namespace) + ): + result = rule_result.findtext("./{0}result".format(namespace)) # find only failed report. if "fail" in result: # get rule corresponding to rule-result - rule = rules[rule_result.attrib['idref']] - title = rule['title'] - description = "\n".join([ - "**IdRef:** `" + rule_result.attrib['idref'] + "`", - "**Title:** `" + title + "`", - ]) + rule = rules[rule_result.attrib["idref"]] + title = rule["title"] + description = "\n".join( + [ + "**IdRef:** `" + rule_result.attrib["idref"] + "`", + "**Title:** `" + title + "`", + ] + ) vulnerability_ids = [] - for vulnerability_id in rule_result.findall("./{0}ident[@system='http://cve.mitre.org']".format(namespace)): + for vulnerability_id in rule_result.findall( + "./{0}ident[@system='http://cve.mitre.org']".format( + namespace + ) + ): vulnerability_ids.append(vulnerability_id.text) # get severity. - severity = rule_result.attrib.get('severity', 'medium').lower().capitalize() + severity = ( + rule_result.attrib.get("severity", "medium") + .lower() + .capitalize() + ) # according to the spec 'unknown' is a possible value - if severity == 'Unknown': - severity = 'Info' + if severity == "Unknown": + severity = "Info" references = "" # get references. - for check_content in rule_result.findall('./{0}check/{0}check-content-ref'.format(namespace)): - references += "**name:** : " + check_content.attrib['name'] + "\n" - references += "**href** : " + check_content.attrib['href'] + "\n" + for check_content in rule_result.findall( + "./{0}check/{0}check-content-ref".format(namespace) + ): + references += ( + "**name:** : " + check_content.attrib["name"] + "\n" + ) + references += ( + "**href** : " + check_content.attrib["href"] + "\n" + ) finding = Finding( title=title, @@ -80,7 +102,7 @@ def get_findings(self, file, test): references=references, dynamic_finding=True, static_finding=False, - unique_id_from_tool=rule_result.attrib['idref'], + unique_id_from_tool=rule_result.attrib["idref"], ) if vulnerability_ids: finding.unsaved_vulnerability_ids = vulnerability_ids @@ -90,13 +112,15 @@ def get_findings(self, file, test): validate_ipv46_address(ip) endpoint = Endpoint(host=ip) except ValidationError: - if '://' in ip: + if "://" in ip: endpoint = Endpoint.from_uri(ip) else: - endpoint = Endpoint.from_uri('//' + ip) + endpoint = Endpoint.from_uri("//" + ip) finding.unsaved_endpoints.append(endpoint) - dupe_key = hashlib.sha256(references.encode('utf-8')).hexdigest() + dupe_key = hashlib.sha256( + references.encode("utf-8") + ).hexdigest() if dupe_key in dupes: find = dupes[dupe_key] if finding.references: @@ -109,5 +133,5 @@ def get_findings(self, file, test): def get_namespace(self, element): """Extract namespace present in XML file.""" - m = re.match(r'\{.*\}', element.tag) - return m.group(0) if m else '' + m = re.match(r"\{.*\}", element.tag) + return m.group(0) if m else "" From 80ffeaeac8f5e68f8918b7fa8196d252ab0d85e3 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:09:43 -0500 Subject: [PATCH 12/29] Update files in folder dojo/tools/openvas_csv with PEP8 standars. --- dojo/tools/openvas_csv/parser.py | 92 +++++++++++++++----------------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/dojo/tools/openvas_csv/parser.py b/dojo/tools/openvas_csv/parser.py index 8dd5cd0e35d..04d6166b231 100644 --- a/dojo/tools/openvas_csv/parser.py +++ b/dojo/tools/openvas_csv/parser.py @@ -1,4 +1,3 @@ - import csv import hashlib import io @@ -9,7 +8,6 @@ class ColumnMappingStrategy(object): - mapped_column = None def __init__(self): @@ -20,25 +18,26 @@ def map_column_value(self, finding, column_value): @staticmethod def evaluate_bool_value(column_value): - if column_value.lower() == 'true': + if column_value.lower() == "true": return True - elif column_value.lower() == 'false': + elif column_value.lower() == "false": return False else: return None def process_column(self, column_name, column_value, finding): - - if column_name.lower() == self.mapped_column and column_value is not None: + if ( + column_name.lower() == self.mapped_column + and column_value is not None + ): self.map_column_value(finding, column_value) elif self.successor is not None: self.successor.process_column(column_name, column_value, finding) class DateColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'timestamp' + self.mapped_column = "timestamp" super(DateColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -46,9 +45,8 @@ def map_column_value(self, finding, column_value): class TitleColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'nvt name' + self.mapped_column = "nvt name" super(TitleColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -56,9 +54,8 @@ def map_column_value(self, finding, column_value): class CweColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'cweid' + self.mapped_column = "cweid" super(CweColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -67,9 +64,8 @@ def map_column_value(self, finding, column_value): class PortColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'port' + self.mapped_column = "port" super(PortColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -78,9 +74,8 @@ def map_column_value(self, finding, column_value): class ProtocolColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'port protocol' + self.mapped_column = "port protocol" super(ProtocolColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -89,20 +84,20 @@ def map_column_value(self, finding, column_value): class IpColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'ip' + self.mapped_column = "ip" super(IpColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): - if not finding.unsaved_endpoints[0].host: # process only if host is not already defined (by field hostname) + if not finding.unsaved_endpoints[ + 0 + ].host: # process only if host is not already defined (by field hostname) finding.unsaved_endpoints[0].host = column_value class HostnameColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'hostname' + self.mapped_column = "hostname" super(HostnameColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -111,27 +106,25 @@ def map_column_value(self, finding, column_value): class SeverityColumnMappingStrategy(ColumnMappingStrategy): - @staticmethod def is_valid_severity(severity): - valid_severity = ('Info', 'Low', 'Medium', 'High', 'Critical') + valid_severity = ("Info", "Low", "Medium", "High", "Critical") return severity in valid_severity def __init__(self): - self.mapped_column = 'severity' + self.mapped_column = "severity" super(SeverityColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): if self.is_valid_severity(column_value): finding.severity = column_value else: - finding.severity = 'Info' + finding.severity = "Info" class DescriptionColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'summary' + self.mapped_column = "summary" super(DescriptionColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -139,9 +132,8 @@ def map_column_value(self, finding, column_value): class MitigationColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'solution' + self.mapped_column = "solution" super(MitigationColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -149,9 +141,8 @@ def map_column_value(self, finding, column_value): class ImpactColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'vulnerability insight' + self.mapped_column = "vulnerability insight" super(ImpactColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -159,9 +150,8 @@ def map_column_value(self, finding, column_value): class ReferencesColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'specific result' + self.mapped_column = "specific result" super(ReferencesColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -169,9 +159,8 @@ def map_column_value(self, finding, column_value): class ActiveColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'active' + self.mapped_column = "active" super(ActiveColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -179,9 +168,8 @@ def map_column_value(self, finding, column_value): class VerifiedColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'verified' + self.mapped_column = "verified" super(VerifiedColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -189,9 +177,8 @@ def map_column_value(self, finding, column_value): class FalsePositiveColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'falsepositive' + self.mapped_column = "falsepositive" super(FalsePositiveColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -199,9 +186,8 @@ def map_column_value(self, finding, column_value): class DuplicateColumnMappingStrategy(ColumnMappingStrategy): - def __init__(self): - self.mapped_column = 'duplicate' + self.mapped_column = "duplicate" super(DuplicateColumnMappingStrategy, self).__init__() def map_column_value(self, finding, column_value): @@ -209,7 +195,6 @@ def map_column_value(self, finding, column_value): class OpenVASCsvParser(object): - def create_chain(self): date_column_strategy = DateColumnMappingStrategy() title_column_strategy = TitleColumnMappingStrategy() @@ -264,15 +249,14 @@ def get_description_for_scan_types(self, scan_type): return "Import OpenVAS Scan in CSV format. Export as CSV Results on OpenVAS." def get_findings(self, filename, test): - column_names = dict() dupes = dict() chain = self.create_chain() content = filename.read() - if type(content) is bytes: - content = content.decode('utf-8') - reader = csv.reader(io.StringIO(content), delimiter=',', quotechar='"') + if isinstance(content, bytes): + content = content.decode("utf-8") + reader = csv.reader(io.StringIO(content), delimiter=",", quotechar='"') row_number = 0 for row in reader: @@ -286,7 +270,9 @@ def get_findings(self, filename, test): column_number = 0 for column in row: - chain.process_column(column_names[column_number], column, finding) + chain.process_column( + column_names[column_number], column, finding + ) column_number += 1 if finding is not None and row_number > 0: @@ -295,7 +281,17 @@ def get_findings(self, filename, test): if finding.description is None: finding.description = "" - key = hashlib.sha256((str(finding.unsaved_endpoints[0]) + '|' + finding.severity + '|' + finding.title + '|' + finding.description).encode('utf-8')).hexdigest() + key = hashlib.sha256( + ( + str(finding.unsaved_endpoints[0]) + + "|" + + finding.severity + + "|" + + finding.title + + "|" + + finding.description + ).encode("utf-8") + ).hexdigest() if key not in dupes: dupes[key] = finding From f864edccc9f86469746ac7b4f204746a733ee4ba Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:10:56 -0500 Subject: [PATCH 13/29] Update files in folder dojo/tools/ort with PEP8 standars. --- dojo/tools/ort/parser.py | 121 ++++++++++++++++++++++----------------- 1 file changed, 68 insertions(+), 53 deletions(-) diff --git a/dojo/tools/ort/parser.py b/dojo/tools/ort/parser.py index 30ffcb853f3..da8f485b46c 100644 --- a/dojo/tools/ort/parser.py +++ b/dojo/tools/ort/parser.py @@ -18,7 +18,6 @@ def get_description_for_scan_types(self, scan_type): return "Import Outpost24 endpoint vulnerability scan in XML format." def get_findings(self, json_output, test): - if json_output is None: return list() @@ -32,27 +31,32 @@ def parse_json(self, json_output): try: data = json_output.read() try: - tree = json.loads(str(data, 'utf-8')) - except: + tree = json.loads(str(data, "utf-8")) + except BaseException: tree = json.loads(data) - except: + except BaseException: raise ValueError("Invalid format") return tree def get_items(self, evaluatedModel, test): items = {} - packages = evaluatedModel['packages'] - dependency_trees = evaluatedModel['dependency_trees'] - rule_violations = evaluatedModel['rule_violations'] - licenses = evaluatedModel['licenses'] - rule_violations_unresolved = get_unresolved_rule_violations(rule_violations) - rule_violations_models = get_rule_violation_models(rule_violations_unresolved, packages, licenses, - dependency_trees) + packages = evaluatedModel["packages"] + dependency_trees = evaluatedModel["dependency_trees"] + rule_violations = evaluatedModel["rule_violations"] + licenses = evaluatedModel["licenses"] + rule_violations_unresolved = get_unresolved_rule_violations( + rule_violations + ) + rule_violations_models = get_rule_violation_models( + rule_violations_unresolved, packages, licenses, dependency_trees + ) for model in rule_violations_models: item = get_item(model, test) - unique_key = hashlib.md5((item.title + item.references).encode()).hexdigest() + unique_key = hashlib.md5( + (item.title + item.references).encode() + ).hexdigest() items[unique_key] = item return list(items.values()) @@ -67,16 +71,16 @@ def get_unresolved_rule_violations(rule_violations): def is_rule_violation_unresolved(rule_violation): - return 'resolutions' not in rule_violation + return "resolutions" not in rule_violation def find_in_dependency_tree(tree, package_id): - if 'pkg' in tree and tree['pkg'] == package_id: + if "pkg" in tree and tree["pkg"] == package_id: return True else: - if 'children' in tree: + if "children" in tree: found_in_child = False - for child in tree['children']: + for child in tree["children"]: if found_in_child: break else: @@ -90,57 +94,69 @@ def get_project_ids_for_package(dependency_trees, package_id): project_ids = [] for project in dependency_trees: if find_in_dependency_tree(project, package_id): - project_ids.append(project['pkg']) + project_ids.append(project["pkg"]) return project_ids def get_name_id_for_package(packages, package__id): name = "" for package in packages: - if package['_id'] == package__id: - name = package['id'] + if package["_id"] == package__id: + name = package["id"] break return name -def get_rule_violation_models(rule_violations_unresolved, packages, licenses, dependency_trees): +def get_rule_violation_models( + rule_violations_unresolved, packages, licenses, dependency_trees +): models = [] for violation in rule_violations_unresolved: - models.append(get_rule_violation_model(violation, packages, licenses, dependency_trees)) + models.append( + get_rule_violation_model( + violation, packages, licenses, dependency_trees + ) + ) return models -def get_rule_violation_model(rule_violation_unresolved, packages, licenses, dependency_trees): - project_ids = get_project_ids_for_package(dependency_trees, rule_violation_unresolved['pkg']) +def get_rule_violation_model( + rule_violation_unresolved, packages, licenses, dependency_trees +): + project_ids = get_project_ids_for_package( + dependency_trees, rule_violation_unresolved["pkg"] + ) project_names = [] for id in project_ids: project_names.append(get_name_id_for_package(packages, id)) - package = find_package_by_id(packages, rule_violation_unresolved['pkg']) - if 'license' in rule_violation_unresolved: - license_tmp = rule_violation_unresolved['license'] + package = find_package_by_id(packages, rule_violation_unresolved["pkg"]) + if "license" in rule_violation_unresolved: + license_tmp = rule_violation_unresolved["license"] else: - license_tmp = 'unset' - if 'license_source' not in rule_violation_unresolved: - rule_violation_unresolved['license_source'] = 'unset' + license_tmp = "unset" + if "license_source" not in rule_violation_unresolved: + rule_violation_unresolved["license_source"] = "unset" license_id = find_license_id(licenses, license_tmp) - return RuleViolationModel(package, license_id, project_names, rule_violation_unresolved) + return RuleViolationModel( + package, license_id, project_names, rule_violation_unresolved + ) def find_package_by_id(packages, pkg_id): package = None for pkg in packages: - if pkg['_id'] == pkg_id: + if pkg["_id"] == pkg_id: package = pkg break return package def find_license_id(licenses, license_id): - id = '' + id = "" for lic in licenses: - if lic['_id'] == license_id: - id = lic['id'] + if lic["_id"] == license_id: + id = lic["id"] break return id @@ -155,12 +171,14 @@ def get_item(model, test): severity = get_severity(model.rule_violation) - finding = Finding(title=model.rule_violation['rule'], - test=test, - references=model.rule_violation['message'], - description=desc, - severity=severity, - static_finding=True) + finding = Finding( + title=model.rule_violation["rule"], + test=test, + references=model.rule_violation["message"], + description=desc, + severity=severity, + static_finding=True, + ) return finding @@ -173,20 +191,17 @@ def get_item(model, test): # projects: [] # rule_violation: dict -RuleViolationModel = namedtuple('RuleViolationModel', [ - 'pkg', - 'license_id', - 'projects', - 'rule_violation' -]) +RuleViolationModel = namedtuple( + "RuleViolationModel", ["pkg", "license_id", "projects", "rule_violation"] +) def get_severity(rule_violation): - if rule_violation['severity'] == 'ERROR': - return 'High' - elif rule_violation['severity'] == 'WARNING': - return 'Medium' - elif rule_violation['severity'] == 'HINT': - return 'Info' + if rule_violation["severity"] == "ERROR": + return "High" + elif rule_violation["severity"] == "WARNING": + return "Medium" + elif rule_violation["severity"] == "HINT": + return "Info" else: - return 'Critical' + return "Critical" From 9029ab6fd575e440ac69bc7018fd1c69e119db75 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:11:44 -0500 Subject: [PATCH 14/29] Update files in folder dojo/tools/ossindex_devaudit with PEP8 standars. --- dojo/tools/ossindex_devaudit/parser.py | 90 ++++++++++++++------------ 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/dojo/tools/ossindex_devaudit/parser.py b/dojo/tools/ossindex_devaudit/parser.py index 7cee84546b0..8d04bac2d48 100644 --- a/dojo/tools/ossindex_devaudit/parser.py +++ b/dojo/tools/ossindex_devaudit/parser.py @@ -20,7 +20,6 @@ def get_description_for_scan_types(self, scan_type): return "Import OssIndex Devaudit SCA Scan in json format." def get_findings(self, json_file, test): - tree = self.parse_json(json_file) if tree: @@ -39,66 +38,77 @@ def parse_json(self, json_file): return tree def get_items(self, tree, test): - items = {} results = {key: value for (key, value) in tree.items()} - for package in results.get('Packages', []): - package_data = package['Package'] - if len(package.get('Vulnerabilities', [])) > 0: - for vulnerability in package.get('Vulnerabilities', []): + for package in results.get("Packages", []): + package_data = package["Package"] + if len(package.get("Vulnerabilities", [])) > 0: + for vulnerability in package.get("Vulnerabilities", []): item = get_item( - dependency_name=package_data['name'], - dependency_version=package_data['version'], - dependency_source=package_data['pm'], + dependency_name=package_data["name"], + dependency_version=package_data["version"], + dependency_source=package_data["pm"], vulnerability=vulnerability, - test=test + test=test, ) - unique_key = vulnerability['id'] + unique_key = vulnerability["id"] items[unique_key] = item return items.values() -def get_item(dependency_name, dependency_version, dependency_source, vulnerability, test): - - cwe_data = vulnerability.get('cwe', 'CWE-1035') - if cwe_data is None or cwe_data.startswith('CWE') is False: - cwe_data = 'CWE-1035' +def get_item( + dependency_name, dependency_version, dependency_source, vulnerability, test +): + cwe_data = vulnerability.get("cwe", "CWE-1035") + if cwe_data is None or cwe_data.startswith("CWE") is False: + cwe_data = "CWE-1035" try: - cwe = int(cwe_data.split('-')[1]) + cwe = int(cwe_data.split("-")[1]) except ValueError: - raise ValueError('Attempting to convert the CWE value to an integer failed') - - finding = Finding(title=dependency_source + ":" + dependency_name + " - " + "(" + dependency_version + ", " + cwe_data + ")", - test=test, - severity=get_severity(vulnerability.get('cvssScore', '')), - description=vulnerability['title'], - cwe=cwe, - cvssv3=vulnerability['cvssVector'].replace('CVSS:3.0', ''), - mitigation='Upgrade the component to the latest non-vulnerable version, or remove the package if it is not in use.', - references=vulnerability.get('reference', ''), - false_p=False, - duplicate=False, - out_of_scope=False, - mitigated=None, - static_finding=False, - dynamic_finding=False, - impact="No impact provided by scan") + raise ValueError( + "Attempting to convert the CWE value to an integer failed" + ) + + finding = Finding( + title=dependency_source + + ":" + + dependency_name + + " - " + + "(" + + dependency_version + + ", " + + cwe_data + + ")", + test=test, + severity=get_severity(vulnerability.get("cvssScore", "")), + description=vulnerability["title"], + cwe=cwe, + cvssv3=vulnerability["cvssVector"].replace("CVSS:3.0", ""), + mitigation="Upgrade the component to the latest non-vulnerable version, or remove the package if it is not in use.", + references=vulnerability.get("reference", ""), + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated=None, + static_finding=False, + dynamic_finding=False, + impact="No impact provided by scan", + ) return finding def get_severity(cvss_score): - - result = 'Info' + result = "Info" if cvss_score != "": ratings = [ - ('Critical', 9.0, 10.0), - ('High', 7.0, 8.9), - ('Medium', 4.0, 6.9), - ('Low', 0.1, 3.9) + ("Critical", 9.0, 10.0), + ("High", 7.0, 8.9), + ("Medium", 4.0, 6.9), + ("Low", 0.1, 3.9), ] for severity, low, high in ratings: From a1a7d2f3b4983feff771e2ee61ddd4ffc6876051 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:12:42 -0500 Subject: [PATCH 15/29] Update files in folder dojo/tools/outpost24 with PEP8 standars. --- dojo/tools/outpost24/parser.py | 74 +++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/dojo/tools/outpost24/parser.py b/dojo/tools/outpost24/parser.py index 13be837541a..8fd244cc425 100644 --- a/dojo/tools/outpost24/parser.py +++ b/dojo/tools/outpost24/parser.py @@ -8,7 +8,6 @@ class Outpost24Parser(object): - def get_scan_types(self): return ["Outpost24 Scan"] @@ -21,55 +20,72 @@ def get_description_for_scan_types(self, scan_type): def get_findings(self, file, test): tree = ElementTree.parse(file) items = list() - for detail in tree.iterfind('.//detaillist/detail'): + for detail in tree.iterfind(".//detaillist/detail"): # finding details - title = detail.findtext('name') + title = detail.findtext("name") # date = detail.findtext('date') # can be used for Finding.date? - vulnerability_id = detail.findtext('./cve/id') - url = detail.findtext('./referencelist/reference/[type=\'solution\']/../url') - description = detail.findtext('description') - mitigation = detail.findtext('solution') - impact = detail.findtext('information') - cvss_score = detail.findtext('cvss_v3_score') or detail.findtext('cvss_score') + vulnerability_id = detail.findtext("./cve/id") + url = detail.findtext( + "./referencelist/reference/[type='solution']/../url" + ) + description = detail.findtext("description") + mitigation = detail.findtext("solution") + impact = detail.findtext("information") + cvss_score = detail.findtext("cvss_v3_score") or detail.findtext( + "cvss_score" + ) if not cvss_score: cvss_score = 0 if cvss_score: score = float(cvss_score) if score < 4: - severity = 'Low' + severity = "Low" elif score < 7: - severity = 'Medium' + severity = "Medium" elif score < 9: - severity = 'High' + severity = "High" else: - severity = 'Critical' + severity = "Critical" else: - risk = int(detail.findtext('risk')) + risk = int(detail.findtext("risk")) if risk == 0: - severity = 'Low' + severity = "Low" elif risk == 1: - severity = 'Medium' + severity = "Medium" elif risk == 2: - severity = 'High' + severity = "High" else: - severity = 'Critical' - cvss_description = detail.findtext('cvss_vector_description') - cvss_vector = detail.findtext('cvss_v3_vector') or detail.findtext('cvss_vector') - severity_justification = "{}\n{}".format(cvss_score, cvss_description) - finding = Finding(title=title, test=test, url=url, description=description, mitigation=mitigation, - impact=impact, severity=severity, - severity_justification=severity_justification) + severity = "Critical" + cvss_description = detail.findtext("cvss_vector_description") + cvss_vector = detail.findtext("cvss_v3_vector") or detail.findtext( + "cvss_vector" + ) + severity_justification = "{}\n{}".format( + cvss_score, cvss_description + ) + finding = Finding( + title=title, + test=test, + url=url, + description=description, + mitigation=mitigation, + impact=impact, + severity=severity, + severity_justification=severity_justification, + ) if vulnerability_id: finding.unsaved_vulnerability_ids = [vulnerability_id] # endpoint details - host = detail.findtext('ip') + host = detail.findtext("ip") if host: - protocol = detail.findtext('./portinfo/service') + protocol = detail.findtext("./portinfo/service") try: - port = int(detail.findtext('./portinfo/portnumber')) - except ValueError as ve: + port = int(detail.findtext("./portinfo/portnumber")) + except ValueError: logger.debug("General port given. Assigning 0 as default.") port = 0 - finding.unsaved_endpoints.append(Endpoint(protocol=protocol, host=host, port=port)) + finding.unsaved_endpoints.append( + Endpoint(protocol=protocol, host=host, port=port) + ) items.append(finding) return items From 12bdaf1811d6be8425427a0863218c18ed4b6906 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:13:49 -0500 Subject: [PATCH 16/29] Update files in folder dojo/tools/php_security_audit_v2 with PEP8 standars. --- dojo/tools/php_security_audit_v2/parser.py | 26 +++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/dojo/tools/php_security_audit_v2/parser.py b/dojo/tools/php_security_audit_v2/parser.py index 4df82d3de56..e93165a14cc 100644 --- a/dojo/tools/php_security_audit_v2/parser.py +++ b/dojo/tools/php_security_audit_v2/parser.py @@ -5,7 +5,6 @@ class PhpSecurityAuditV2Parser(object): - def get_scan_types(self): return ["PHP Security Audit v2"] @@ -18,8 +17,8 @@ def get_description_for_scan_types(self, scan_type): def get_findings(self, filename, test): tree = filename.read() try: - data = json.loads(str(tree, 'utf-8')) - except: + data = json.loads(str(tree, "utf-8")) + except BaseException: data = json.loads(tree) dupes = dict() @@ -36,9 +35,16 @@ def get_findings(self, filename, test): findingdetail += "Rule Source: " + issue["source"] + "\n" findingdetail += "Details: " + issue["message"] + "\n" - sev = PhpSecurityAuditV2Parser.get_severity_word(issue["severity"]) + sev = PhpSecurityAuditV2Parser.get_severity_word( + issue["severity"] + ) - dupe_key = title + filepath + str(issue["line"]) + str(issue["column"]) + dupe_key = ( + title + + filepath + + str(issue["line"]) + + str(issue["column"]) + ) if dupe_key in dupes: find = dupes[dupe_key] @@ -57,7 +63,7 @@ def get_findings(self, filename, test): ) dupes[dupe_key] = find - findingdetail = '' + findingdetail = "" return list(dupes.values()) @@ -66,10 +72,10 @@ def get_severity_word(severity): sev = math.ceil(severity / 2) if sev == 5: - return 'Critical' + return "Critical" elif sev == 4: - return 'High' + return "High" elif sev == 3: - return 'Medium' + return "Medium" else: - return 'Low' + return "Low" From 81f36efc94a7a7601a6dbaeb01e6f2c0f987cebd Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:14:41 -0500 Subject: [PATCH 17/29] Update files in folder dojo/tools/php_symfony_security_check with PEP8 standars. --- .../php_symfony_security_check/parser.py | 74 +++++++++++-------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/dojo/tools/php_symfony_security_check/parser.py b/dojo/tools/php_symfony_security_check/parser.py index fbc2e8d8b54..e00b9ac975b 100644 --- a/dojo/tools/php_symfony_security_check/parser.py +++ b/dojo/tools/php_symfony_security_check/parser.py @@ -4,7 +4,6 @@ class PhpSymfonySecurityCheckParser(object): - def get_scan_types(self): return ["PHP Symfony Security Check"] @@ -24,10 +23,10 @@ def parse_json(self, json_file): try: data = json_file.read() try: - tree = json.loads(str(data, 'utf-8')) - except: + tree = json.loads(str(data, "utf-8")) + except BaseException: tree = json.loads(data) - except: + except BaseException: raise Exception("Invalid format") return tree @@ -36,41 +35,54 @@ def get_items(self, tree, test): items = {} for dependency_name, dependency_data in list(tree.items()): - advisories = dependency_data.get('advisories') - dependency_version = dependency_data['version'] - if dependency_version and dependency_version.startswith('v'): + advisories = dependency_data.get("advisories") + dependency_version = dependency_data["version"] + if dependency_version and dependency_version.startswith("v"): dependency_version = dependency_version[1:] for advisory in advisories: - item = get_item(dependency_name, dependency_version, advisory, test) - unique_key = str(dependency_name) + str(dependency_data['version'] + str(advisory['cve'])) + item = get_item( + dependency_name, dependency_version, advisory, test + ) + unique_key = str(dependency_name) + str( + dependency_data["version"] + str(advisory["cve"]) + ) items[unique_key] = item return list(items.values()) def get_item(dependency_name, dependency_version, advisory, test): - - finding = Finding(title=dependency_name + " - " + "(" + dependency_version + ", " + advisory['cve'] + ")", - test=test, - # TODO decide how to handle the fact we don't have a severity. None will lead to problems handling minimum severity on import - severity='Info', - description=advisory['title'], - # TODO Decide if the default '1035: vulnerable 3rd party component' is OK to use? - cwe=1035, - mitigation='upgrade', - references=advisory['link'], - false_p=False, - duplicate=False, - out_of_scope=False, - mitigated=None, - impact="No impact provided", - static_finding=True, - dynamic_finding=False, - component_name=dependency_name, - component_version=dependency_version) - - if advisory['cve']: - finding.unsaved_vulnerability_ids = [advisory['cve']] + finding = Finding( + title=dependency_name + + " - " + + "(" + + dependency_version + + ", " + + advisory["cve"] + + ")", + test=test, + # TODO decide how to handle the fact we don't have a severity. None + # will lead to problems handling minimum severity on import + severity="Info", + description=advisory["title"], + # TODO Decide if the default '1035: vulnerable 3rd party component' is + # OK to use? + cwe=1035, + mitigation="upgrade", + references=advisory["link"], + false_p=False, + duplicate=False, + out_of_scope=False, + mitigated=None, + impact="No impact provided", + static_finding=True, + dynamic_finding=False, + component_name=dependency_name, + component_version=dependency_version, + ) + + if advisory["cve"]: + finding.unsaved_vulnerability_ids = [advisory["cve"]] return finding From 187bec2b979519edd1ae9d512116d86e0ce3acc5 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:15:32 -0500 Subject: [PATCH 18/29] Update files in folder dojo/tools/pip_audit with PEP8 standars. --- dojo/tools/pip_audit/parser.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/dojo/tools/pip_audit/parser.py b/dojo/tools/pip_audit/parser.py index 7c2871a05c3..726667987fb 100644 --- a/dojo/tools/pip_audit/parser.py +++ b/dojo/tools/pip_audit/parser.py @@ -4,7 +4,6 @@ class PipAuditParser: - def get_scan_types(self): return ["pip-audit Scan"] @@ -18,39 +17,40 @@ def requires_file(self, scan_type): return True def get_findings(self, scan_file, test): - data = json.load(scan_file) findings = list() for item in data: - vulnerabilities = item.get('vulns', []) + vulnerabilities = item.get("vulns", []) if vulnerabilities: - component_name = item['name'] - component_version = item.get('version') + component_name = item["name"] + component_version = item.get("version") for vulnerability in vulnerabilities: - vuln_id = vulnerability.get('id') - vuln_fix_versions = vulnerability.get('fix_versions') - vuln_description = vulnerability.get('description') + vuln_id = vulnerability.get("id") + vuln_fix_versions = vulnerability.get("fix_versions") + vuln_description = vulnerability.get("description") - title = f'{vuln_id} in {component_name}:{component_version}' + title = ( + f"{vuln_id} in {component_name}:{component_version}" + ) - description = '' + description = "" description += vuln_description mitigation = None if vuln_fix_versions: - mitigation = 'Upgrade to version:' + mitigation = "Upgrade to version:" if len(vuln_fix_versions) == 1: - mitigation += f' {vuln_fix_versions[0]}' + mitigation += f" {vuln_fix_versions[0]}" else: for fix_version in vuln_fix_versions: - mitigation += f'\n- {fix_version}' + mitigation += f"\n- {fix_version}" finding = Finding( test=test, title=title, cwe=1352, - severity='Medium', + severity="Medium", description=description, mitigation=mitigation, component_name=component_name, From a9c122badcbde72c6115e7c12d1e84979a3170cf Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:16:27 -0500 Subject: [PATCH 19/29] Update files in folder dojo/tools/pmd with PEP8 standars. --- dojo/tools/pmd/parser.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/dojo/tools/pmd/parser.py b/dojo/tools/pmd/parser.py index 22296ebe8cf..d3f8c5eda2e 100644 --- a/dojo/tools/pmd/parser.py +++ b/dojo/tools/pmd/parser.py @@ -5,7 +5,6 @@ class PmdParser(object): - def get_scan_types(self): return ["PMD Scan"] @@ -19,9 +18,11 @@ def get_findings(self, filename, test): dupes = dict() content = filename.read() - if type(content) is bytes: - content = content.decode('utf-8') - reader = list(csv.DictReader(io.StringIO(content), delimiter=',', quotechar='"')) + if isinstance(content, bytes): + content = content.decode("utf-8") + reader = list( + csv.DictReader(io.StringIO(content), delimiter=",", quotechar='"') + ) for row in reader: finding = Finding(test=test) @@ -40,7 +41,9 @@ def get_findings(self, filename, test): priority = "Info" finding.severity = priority - description = "Description: {}\n".format(row['Description'].strip()) + description = "Description: {}\n".format( + row["Description"].strip() + ) description += "Rule set: {}\n".format(row["Rule set"].strip()) description += "Problem: {}\n".format(row["Problem"].strip()) description += "Package: {}\n".format(row["Package"].strip()) @@ -50,12 +53,16 @@ def get_findings(self, filename, test): finding.impact = "No impact provided" finding.mitigation = "No mitigation provided" - key = hashlib.sha256("|".join([ - finding.title, - finding.description, - finding.file_path, - finding.line - ]).encode("utf-8")).hexdigest() + key = hashlib.sha256( + "|".join( + [ + finding.title, + finding.description, + finding.file_path, + finding.line, + ] + ).encode("utf-8") + ).hexdigest() if key not in dupes: dupes[key] = finding From 8ed31471ac6b6a4be5677d84a1ca6d0bc39b4a5e Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:17:10 -0500 Subject: [PATCH 20/29] Update files in folder dojo/tools/popeye with PEP8 standars. --- dojo/tools/popeye/parser.py | 47 +++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/dojo/tools/popeye/parser.py b/dojo/tools/popeye/parser.py index 6c49a27fb5b..67e176a9110 100644 --- a/dojo/tools/popeye/parser.py +++ b/dojo/tools/popeye/parser.py @@ -22,20 +22,41 @@ def get_findings(self, file, test): data = json.load(file) dupes = dict() - for sanitizer in data['popeye']['sanitizers']: - issues = sanitizer.get('issues') + for sanitizer in data["popeye"]["sanitizers"]: + issues = sanitizer.get("issues") if issues: for issue_group, issue_list in issues.items(): for issue in issue_list: - if issue['level'] != 0: - title = sanitizer['sanitizer'] + " " + issue_group + " " + issue['message'] - severity = self.get_defect_dojo_severity(issue['level']) - description = "**Sanitizer** : " + sanitizer['sanitizer'] + "\n\n" + \ - "**Resource** : " + issue_group + "\n\n" + \ - "**Group** : " + issue['group'] + "\n\n" + \ - "**Severity** : " + self.get_popeye_level_string(issue['level']) + "\n\n" + \ - "**Message** : " + issue['message'] - vuln_id_from_tool = re.search(r'\[(POP-\d+)\].+', issue['message']).group(1) + if issue["level"] != 0: + title = ( + sanitizer["sanitizer"] + + " " + + issue_group + + " " + + issue["message"] + ) + severity = self.get_defect_dojo_severity( + issue["level"] + ) + description = ( + "**Sanitizer** : " + + sanitizer["sanitizer"] + + "\n\n" + + "**Resource** : " + + issue_group + + "\n\n" + + "**Group** : " + + issue["group"] + + "\n\n" + + "**Severity** : " + + self.get_popeye_level_string(issue["level"]) + + "\n\n" + + "**Message** : " + + issue["message"] + ) + vuln_id_from_tool = re.search( + r"\[(POP-\d+)\].+", issue["message"] + ).group(1) finding = Finding( title=title, test=test, @@ -46,7 +67,9 @@ def get_findings(self, file, test): vuln_id_from_tool=vuln_id_from_tool, ) # internal de-duplication - dupe_key = hashlib.sha256(str(description + title).encode('utf-8')).hexdigest() + dupe_key = hashlib.sha256( + str(description + title).encode("utf-8") + ).hexdigest() if dupe_key not in dupes: dupes[dupe_key] = finding return list(dupes.values()) From 473b7754e07a3d0f82c41d400d7da73e181294d6 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:18:39 -0500 Subject: [PATCH 21/29] Update files in folder dojo/tools/pwn_sast with PEP8 standars. --- dojo/tools/pwn_sast/parser.py | 65 +++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/dojo/tools/pwn_sast/parser.py b/dojo/tools/pwn_sast/parser.py index d25ebaff629..f86b8cbd2a7 100644 --- a/dojo/tools/pwn_sast/parser.py +++ b/dojo/tools/pwn_sast/parser.py @@ -19,17 +19,16 @@ def get_description_for_scan_types(self, scan_type): return "Import pwn_sast Driver findings in JSON format." def get_findings(self, filename, test): - results = json.load(filename) if results is not None: - report_name = results.get("report_name") + results.get("report_name") data_arr = results.get("data") findings = {} for data_hash in data_arr: - timestamp = data_hash.get("timestamp") + data_hash.get("timestamp") security_references = data_hash.get("security_references") if security_references is not None: @@ -54,37 +53,45 @@ def get_findings(self, filename, test): offending_file = None line_no_and_contents = data_hash.get("line_no_and_contents") - test_case_filter = data_hash.get("test_case_filter") - steps_to_reproduce = "\n".join([ - "Install pwn_sast Driver via: https://github.com/0dayinc/pwn#installation", - "Execute the pwn_sast Driver via:", - f"```pwn_sast --dir-path . --uri-source-root {git_repo_root_uri} -s```" - ]) + data_hash.get("test_case_filter") + steps_to_reproduce = "\n".join( + [ + "Install pwn_sast Driver via: https://github.com/0dayinc/pwn#installation", + "Execute the pwn_sast Driver via:", + f"```pwn_sast --dir-path . --uri-source-root {git_repo_root_uri} -s```", + ] + ) for line in line_no_and_contents: offending_uri = f"{git_repo_root_uri}/{offending_file}" line_no = line.get("line_no") contents = line.get("contents") author = line.get("author") - severity = 'Info' - description = "\n".join([ - f"SAST Module: {sast_module}", - f"Offending URI: {offending_uri}", - f"Line: {line_no}", - f"Committed By: {author}", - "Line Contents:", - f"```{contents}```" - ]) - - impact = "\n".join([ - f"Security Control Impacted: {section}", - f"NIST 800-53 Security Control Details: {nist_800_53_uri}", - f"CWE Details: {cwe_uri}" - ]) - - mitigation = "\n".join([ - f"NIST 800-53 Security Control Details / Mitigation Strategy: {nist_800_53_uri}", - ]) + severity = "Info" + description = "\n".join( + [ + f"SAST Module: {sast_module}", + f"Offending URI: {offending_uri}", + f"Line: {line_no}", + f"Committed By: {author}", + "Line Contents:", + f"```{contents}```", + ] + ) + + impact = "\n".join( + [ + f"Security Control Impacted: {section}", + f"NIST 800-53 Security Control Details: {nist_800_53_uri}", + f"CWE Details: {cwe_uri}", + ] + ) + + mitigation = "\n".join( + [ + f"NIST 800-53 Security Control Details / Mitigation Strategy: {nist_800_53_uri}", + ] + ) unique_finding_key = hashlib.sha256( (offending_uri + contents).encode("utf-8") @@ -106,7 +113,7 @@ def get_findings(self, filename, test): cwe=cwe_id, nb_occurences=1, steps_to_reproduce=steps_to_reproduce, - file_path=offending_file + file_path=offending_file, ) findings[unique_finding_key] = finding From 6b8b6db7342f44d5bc896f152dc5627a725ce134 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:19:38 -0500 Subject: [PATCH 22/29] Update files in folder dojo/tools/qualys with PEP8 standars. --- dojo/tools/qualys/csv_parser.py | 87 ++++++---- dojo/tools/qualys/parser.py | 296 +++++++++++++++++--------------- 2 files changed, 208 insertions(+), 175 deletions(-) diff --git a/dojo/tools/qualys/csv_parser.py b/dojo/tools/qualys/csv_parser.py index e7377153ddb..e210c7aea9c 100644 --- a/dojo/tools/qualys/csv_parser.py +++ b/dojo/tools/qualys/csv_parser.py @@ -19,11 +19,9 @@ def parse_csv(csv_file) -> [Finding]: content = csv_file.read() if isinstance(content, bytes): - content = content.decode('utf-8') + content = content.decode("utf-8") csv_reader = csv.DictReader( - io.StringIO(content), - delimiter=',', - quotechar='"' + io.StringIO(content), delimiter=",", quotechar='"' ) report_findings = get_report_findings(csv_reader) @@ -45,7 +43,7 @@ def get_report_findings(csv_reader) -> [dict]: report_findings = [] for row in csv_reader: - if row.get('Title') and row['Title'] != 'Title': + if row.get("Title") and row["Title"] != "Title": report_findings.append(row) return report_findings @@ -64,27 +62,31 @@ def _extract_cvss_vectors(cvss_base, cvss_temporal): A CVSS3 Vector including both Base and Temporal if available """ - vector_pattern = r'^\d{1,2}.\d \((.*)\)' - cvss_vector = 'CVSS:3.0/' + vector_pattern = r"^\d{1,2}.\d \((.*)\)" + cvss_vector = "CVSS:3.0/" if cvss_base: try: cvss_vector += re.search(vector_pattern, cvss_base).group(1) except IndexError: - _logger.error(f'CVSS3 Base Vector not found in {cvss_base}') + _logger.error(f"CVSS3 Base Vector not found in {cvss_base}") except AttributeError: - _logger.error(f'CVSS3 Base Vector not found in {cvss_base}') + _logger.error(f"CVSS3 Base Vector not found in {cvss_base}") if cvss_temporal: try: - cvss_temporal_vector = re.search(vector_pattern, cvss_temporal).group(1) - cvss_vector += '/' + cvss_temporal_vector = re.search( + vector_pattern, cvss_temporal + ).group(1) + cvss_vector += "/" cvss_vector += cvss_temporal_vector except IndexError: _logger.error( - f'CVSS3 Temporal Vector not found in {cvss_base}') + f"CVSS3 Temporal Vector not found in {cvss_base}" + ) except AttributeError: _logger.error( - f'CVSS3 Temporal Vector not found in {cvss_base}') + f"CVSS3 Temporal Vector not found in {cvss_base}" + ) return cvss_vector @@ -98,42 +100,55 @@ def build_findings_from_dict(report_findings: [dict]) -> [Finding]: """ severity_lookup = { - '1': 'Info', - '2': 'Low', - '3': 'Medium', - '4': 'High', - '5': 'Critical'} + "1": "Info", + "2": "Low", + "3": "Medium", + "4": "High", + "5": "Critical", + } dojo_findings = [] for report_finding in report_findings: - if report_finding.get('FQDN'): - endpoint = Endpoint.from_uri(report_finding.get('FQDN')) + if report_finding.get("FQDN"): + endpoint = Endpoint.from_uri(report_finding.get("FQDN")) else: - endpoint = Endpoint(host=report_finding['IP']) + endpoint = Endpoint(host=report_finding["IP"]) finding = Finding( title=f"QID-{report_finding['QID']} | {report_finding['Title']}", - mitigation=report_finding['Solution'], + mitigation=report_finding["Solution"], description=f"{report_finding['Threat']}\nResult Evidence: \n{report_finding.get('Threat', 'Not available')}", - severity=severity_lookup.get(report_finding['Severity'], 'Info'), - impact=report_finding['Impact'], - date=datetime.strptime(report_finding['Last Detected'], "%m/%d/%Y %H:%M:%S").date(), - vuln_id_from_tool=report_finding['QID'], + severity=severity_lookup.get(report_finding["Severity"], "Info"), + impact=report_finding["Impact"], + date=datetime.strptime( + report_finding["Last Detected"], "%m/%d/%Y %H:%M:%S" + ).date(), + vuln_id_from_tool=report_finding["QID"], cvssv3=_extract_cvss_vectors( - report_finding['CVSS3 Base'], - report_finding['CVSS3 Temporal'])) - - cve_data = report_finding.get('CVE ID') - finding.unsaved_vulnerability_ids = cve_data.split(',') if ',' in cve_data else [cve_data] - - # Qualys reports regression findings as active, but with a Date Last Fixed. - if report_finding['Date Last Fixed']: - finding.mitigated = datetime.strptime(report_finding['Date Last Fixed'], "%m/%d/%Y %H:%M:%S") + report_finding["CVSS3 Base"], report_finding["CVSS3 Temporal"] + ), + ) + + cve_data = report_finding.get("CVE ID") + finding.unsaved_vulnerability_ids = ( + cve_data.split(",") if "," in cve_data else [cve_data] + ) + + # Qualys reports regression findings as active, but with a Date Last + # Fixed. + if report_finding["Date Last Fixed"]: + finding.mitigated = datetime.strptime( + report_finding["Date Last Fixed"], "%m/%d/%Y %H:%M:%S" + ) finding.is_mitigated = True else: finding.is_mitigated = False - finding.active = report_finding['Vuln Status'] in ('Active', 'Re-Opened', 'New') + finding.active = report_finding["Vuln Status"] in ( + "Active", + "Re-Opened", + "New", + ) if finding.active: finding.mitigated = None diff --git a/dojo/tools/qualys/parser.py b/dojo/tools/qualys/parser.py index a757cb4733f..d86c7f4c50c 100644 --- a/dojo/tools/qualys/parser.py +++ b/dojo/tools/qualys/parser.py @@ -10,41 +10,43 @@ logger = logging.getLogger(__name__) -CUSTOM_HEADERS = {'CVSS_score': 'CVSS Score', - 'ip_address': 'IP Address', - 'fqdn': 'FQDN', - 'os': 'OS', - 'port_status': 'Port', - 'vuln_name': 'Vulnerability', - 'vuln_description': 'Description', - 'solution': 'Solution', - 'links': 'Links', - 'cve': 'CVE', - 'vuln_severity': 'Severity', - 'QID': 'QID', - 'first_found': 'First Found', - 'last_found': 'Last Found', - 'found_times': 'Found Times', - 'category': 'Category' - } - -REPORT_HEADERS = ['CVSS_score', - 'ip_address', - 'fqdn', - 'os', - 'port_status', - 'vuln_name', - 'vuln_description', - 'solution', - 'links', - 'cve', - 'Severity', - 'QID', - 'first_found', - 'last_found', - 'found_times', - 'category', - ] +CUSTOM_HEADERS = { + "CVSS_score": "CVSS Score", + "ip_address": "IP Address", + "fqdn": "FQDN", + "os": "OS", + "port_status": "Port", + "vuln_name": "Vulnerability", + "vuln_description": "Description", + "solution": "Solution", + "links": "Links", + "cve": "CVE", + "vuln_severity": "Severity", + "QID": "QID", + "first_found": "First Found", + "last_found": "Last Found", + "found_times": "Found Times", + "category": "Category", +} + +REPORT_HEADERS = [ + "CVSS_score", + "ip_address", + "fqdn", + "os", + "port_status", + "vuln_name", + "vuln_description", + "solution", + "links", + "cve", + "Severity", + "QID", + "first_found", + "last_found", + "found_times", + "category", +] def htmltext(blob): @@ -59,11 +61,13 @@ def split_cvss(value, _temp): return if len(value) > 4: split = value.split(" (") - _temp['CVSS_value'] = float(split[0]) + _temp["CVSS_value"] = float(split[0]) # remove ")" at the end - _temp['CVSS_vector'] = CVSS3("CVSS:3.0/" + split[1][:-1]).clean_vector() + _temp["CVSS_vector"] = CVSS3( + "CVSS:3.0/" + split[1][:-1] + ).clean_vector() else: - _temp['CVSS_value'] = float(value) + _temp["CVSS_value"] = float(value) def parse_finding(host, tree): @@ -71,158 +75,173 @@ def parse_finding(host, tree): issue_row = {} # IP ADDRESS - issue_row['ip_address'] = host.findtext('IP') + issue_row["ip_address"] = host.findtext("IP") # FQDN - issue_row['fqdn'] = host.findtext('DNS') + issue_row["fqdn"] = host.findtext("DNS") # Create Endpoint - if issue_row['fqdn']: - ep = Endpoint(host=issue_row['fqdn']) + if issue_row["fqdn"]: + ep = Endpoint(host=issue_row["fqdn"]) else: - ep = Endpoint(host=issue_row['ip_address']) + ep = Endpoint(host=issue_row["ip_address"]) # OS NAME - issue_row['os'] = host.findtext('OPERATING_SYSTEM') + issue_row["os"] = host.findtext("OPERATING_SYSTEM") # Scan details - for vuln_details in host.iterfind('VULN_INFO_LIST/VULN_INFO'): + for vuln_details in host.iterfind("VULN_INFO_LIST/VULN_INFO"): _temp = issue_row # Port - _gid = vuln_details.find('QID').attrib['id'] - _port = vuln_details.findtext('PORT') - _temp['port_status'] = _port - - _category = str(vuln_details.findtext('CATEGORY')) - _result = str(vuln_details.findtext('RESULT')) - _first_found = str(vuln_details.findtext('FIRST_FOUND')) - _last_found = str(vuln_details.findtext('LAST_FOUND')) - _times_found = str(vuln_details.findtext('TIMES_FOUND')) - - _temp['date'] = datetime.datetime.strptime(vuln_details.findtext('LAST_FOUND'), "%Y-%m-%dT%H:%M:%SZ").date() + _gid = vuln_details.find("QID").attrib["id"] + _port = vuln_details.findtext("PORT") + _temp["port_status"] = _port + + _category = str(vuln_details.findtext("CATEGORY")) + _result = str(vuln_details.findtext("RESULT")) + _first_found = str(vuln_details.findtext("FIRST_FOUND")) + _last_found = str(vuln_details.findtext("LAST_FOUND")) + _times_found = str(vuln_details.findtext("TIMES_FOUND")) + + _temp["date"] = datetime.datetime.strptime( + vuln_details.findtext("LAST_FOUND"), "%Y-%m-%dT%H:%M:%SZ" + ).date() # Vuln_status - status = vuln_details.findtext('VULN_STATUS') + status = vuln_details.findtext("VULN_STATUS") if status == "Active" or status == "Re-Opened" or status == "New": - _temp['active'] = True - _temp['mitigated'] = False - _temp['mitigation_date'] = None + _temp["active"] = True + _temp["mitigated"] = False + _temp["mitigation_date"] = None else: - _temp['active'] = False - _temp['mitigated'] = True - last_fixed = vuln_details.findtext('LAST_FIXED') + _temp["active"] = False + _temp["mitigated"] = True + last_fixed = vuln_details.findtext("LAST_FIXED") if last_fixed is not None: - _temp['mitigation_date'] = datetime.datetime.strptime(last_fixed, "%Y-%m-%dT%H:%M:%SZ").date() + _temp["mitigation_date"] = datetime.datetime.strptime( + last_fixed, "%Y-%m-%dT%H:%M:%SZ" + ).date() else: - _temp['mitigation_date'] = None + _temp["mitigation_date"] = None # read cvss value if present - cvss3 = vuln_details.findtext('CVSS3_FINAL') + cvss3 = vuln_details.findtext("CVSS3_FINAL") if cvss3 is not None and cvss3 != "-": split_cvss(cvss3, _temp) else: - cvss2 = vuln_details.findtext('CVSS_FINAL') + cvss2 = vuln_details.findtext("CVSS_FINAL") if cvss2 is not None and cvss2 != "-": split_cvss(cvss2, _temp) # DefectDojo does not support cvssv2 - _temp['CVSS_vector'] = None + _temp["CVSS_vector"] = None - search = ".//GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS[@id='{}']".format(_gid) + search = ".//GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS[@id='{}']".format( + _gid + ) vuln_item = tree.find(search) if vuln_item is not None: finding = Finding() # Vuln name - _temp['vuln_name'] = vuln_item.findtext('TITLE') + _temp["vuln_name"] = vuln_item.findtext("TITLE") # Vuln Description - _description = str(vuln_item.findtext('THREAT')) + _description = str(vuln_item.findtext("THREAT")) # Solution Strips Heading Workaround(s) # _temp['solution'] = re.sub('Workaround(s)?:.+\n', '', htmltext(vuln_item.findtext('SOLUTION'))) - _temp['solution'] = htmltext(vuln_item.findtext('SOLUTION')) + _temp["solution"] = htmltext(vuln_item.findtext("SOLUTION")) # Vuln_description - _temp['vuln_description'] = "\n".join([htmltext(_description), - htmltext("Category: " + _category), - htmltext("QID: " + str(_gid)), - htmltext("Port: " + str(_port)), - htmltext("Result Evidence: " + _result), - htmltext("First Found: " + _first_found), - htmltext("Last Found: " + _last_found), - htmltext("Times Found: " + _times_found), - ]) + _temp["vuln_description"] = "\n".join( + [ + htmltext(_description), + htmltext("Category: " + _category), + htmltext("QID: " + str(_gid)), + htmltext("Port: " + str(_port)), + htmltext("Result Evidence: " + _result), + htmltext("First Found: " + _first_found), + htmltext("Last Found: " + _last_found), + htmltext("Times Found: " + _times_found), + ] + ) # Impact description - _temp['IMPACT'] = htmltext(vuln_item.findtext('IMPACT')) + _temp["IMPACT"] = htmltext(vuln_item.findtext("IMPACT")) # read cvss value if present and not already read from vuln - if _temp.get('CVSS_value') is None: - cvss3 = vuln_item.findtext('CVSS3_SCORE/CVSS3_BASE') - cvss2 = vuln_item.findtext('CVSS_SCORE/CVSS_BASE') + if _temp.get("CVSS_value") is None: + cvss3 = vuln_item.findtext("CVSS3_SCORE/CVSS3_BASE") + cvss2 = vuln_item.findtext("CVSS_SCORE/CVSS_BASE") if cvss3 is not None and cvss3 != "-": split_cvss(cvss3, _temp) else: - cvss2 = vuln_item.findtext('CVSS_FINAL') + cvss2 = vuln_item.findtext("CVSS_FINAL") if cvss2 is not None and cvss2 != "-": split_cvss(cvss2, _temp) # DefectDojo does not support cvssv2 - _temp['CVSS_vector'] = None + _temp["CVSS_vector"] = None # CVE and LINKS - _temp_cve_details = vuln_item.iterfind('CVE_ID_LIST/CVE_ID') + _temp_cve_details = vuln_item.iterfind("CVE_ID_LIST/CVE_ID") if _temp_cve_details: - _cl = {cve_detail.findtext('ID'): cve_detail.findtext('URL') for cve_detail in _temp_cve_details} - _temp['cve'] = "\n".join(list(_cl.keys())) - _temp['links'] = "\n".join(list(_cl.values())) + _cl = { + cve_detail.findtext("ID"): cve_detail.findtext("URL") + for cve_detail in _temp_cve_details + } + _temp["cve"] = "\n".join(list(_cl.keys())) + _temp["links"] = "\n".join(list(_cl.values())) # The CVE in Qualys report might not have a CVSS score, so findings are informational by default - # unless we can find map to a Severity OR a CVSS score from the findings detail. + # unless we can find map to a Severity OR a CVSS score from the + # findings detail. sev = None - if _temp.get('CVSS_value') is not None and _temp['CVSS_value'] > 0: - if 0.1 <= float(_temp['CVSS_value']) <= 3.9: - sev = 'Low' - elif 4.0 <= float(_temp['CVSS_value']) <= 6.9: - sev = 'Medium' - elif 7.0 <= float(_temp['CVSS_value']) <= 8.9: - sev = 'High' - elif float(_temp['CVSS_value']) >= 9.0: - sev = 'Critical' - elif vuln_item.findtext('SEVERITY') is not None: - if int(vuln_item.findtext('SEVERITY')) == 1: - sev = 'Informational' - elif int(vuln_item.findtext('SEVERITY')) == 2: - sev = 'Low' - elif int(vuln_item.findtext('SEVERITY')) == 3: - sev = 'Medium' - elif int(vuln_item.findtext('SEVERITY')) == 4: - sev = 'High' - elif int(vuln_item.findtext('SEVERITY')) == 5: - sev = 'Critical' + if _temp.get("CVSS_value") is not None and _temp["CVSS_value"] > 0: + if 0.1 <= float(_temp["CVSS_value"]) <= 3.9: + sev = "Low" + elif 4.0 <= float(_temp["CVSS_value"]) <= 6.9: + sev = "Medium" + elif 7.0 <= float(_temp["CVSS_value"]) <= 8.9: + sev = "High" + elif float(_temp["CVSS_value"]) >= 9.0: + sev = "Critical" + elif vuln_item.findtext("SEVERITY") is not None: + if int(vuln_item.findtext("SEVERITY")) == 1: + sev = "Informational" + elif int(vuln_item.findtext("SEVERITY")) == 2: + sev = "Low" + elif int(vuln_item.findtext("SEVERITY")) == 3: + sev = "Medium" + elif int(vuln_item.findtext("SEVERITY")) == 4: + sev = "High" + elif int(vuln_item.findtext("SEVERITY")) == 5: + sev = "Critical" elif sev is None: - sev = 'Informational' + sev = "Informational" finding = None if _temp_cve_details: refs = "\n".join(list(_cl.values())) - finding = Finding(title="QID-" + _gid[4:] + " | " + _temp['vuln_name'], - mitigation=_temp['solution'], - description=_temp['vuln_description'], - severity=sev, - references=refs, - impact=_temp['IMPACT'], - date=_temp['date'], - vuln_id_from_tool=_gid, - ) + finding = Finding( + title="QID-" + _gid[4:] + " | " + _temp["vuln_name"], + mitigation=_temp["solution"], + description=_temp["vuln_description"], + severity=sev, + references=refs, + impact=_temp["IMPACT"], + date=_temp["date"], + vuln_id_from_tool=_gid, + ) else: - finding = Finding(title="QID-" + _gid[4:] + " | " + _temp['vuln_name'], - mitigation=_temp['solution'], - description=_temp['vuln_description'], - severity=sev, - references=_gid, - impact=_temp['IMPACT'], - date=_temp['date'], - vuln_id_from_tool=_gid, - ) - finding.mitigated = _temp['mitigation_date'] - finding.is_mitigated = _temp['mitigated'] - finding.active = _temp['active'] - if _temp.get('CVSS_vector') is not None: - finding.cvssv3 = _temp.get('CVSS_vector') + finding = Finding( + title="QID-" + _gid[4:] + " | " + _temp["vuln_name"], + mitigation=_temp["solution"], + description=_temp["vuln_description"], + severity=sev, + references=_gid, + impact=_temp["IMPACT"], + date=_temp["date"], + vuln_id_from_tool=_gid, + ) + finding.mitigated = _temp["mitigation_date"] + finding.is_mitigated = _temp["mitigated"] + finding.active = _temp["active"] + if _temp.get("CVSS_vector") is not None: + finding.cvssv3 = _temp.get("CVSS_vector") finding.verified = True finding.unsaved_endpoints = list() finding.unsaved_endpoints.append(ep) @@ -233,7 +252,7 @@ def parse_finding(host, tree): def qualys_parser(qualys_xml_file): parser = etree.XMLParser() tree = etree.parse(qualys_xml_file, parser) - host_list = tree.find('HOST_LIST') + host_list = tree.find("HOST_LIST") finding_list = [] if host_list is not None: for host in host_list: @@ -242,7 +261,6 @@ def qualys_parser(qualys_xml_file): class QualysParser(object): - def get_scan_types(self): return ["Qualys Scan"] @@ -253,7 +271,7 @@ def get_description_for_scan_types(self, scan_type): return "Qualys WebGUI output files can be imported in XML format." def get_findings(self, file, test): - if file.name.lower().endswith('.csv'): + if file.name.lower().endswith(".csv"): return csv_parser.parse_csv(file) else: return qualys_parser(file) From faa68f742e4c64e99a69f26e2c214667e1f08aae Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:20:37 -0500 Subject: [PATCH 23/29] Update files in folder dojo/tools/qualys_infrascan_webgui with PEP8 standars. --- dojo/tools/qualys_infrascan_webgui/parser.py | 124 ++++++++++--------- 1 file changed, 68 insertions(+), 56 deletions(-) diff --git a/dojo/tools/qualys_infrascan_webgui/parser.py b/dojo/tools/qualys_infrascan_webgui/parser.py index 29c16742e64..e60084619a7 100644 --- a/dojo/tools/qualys_infrascan_webgui/parser.py +++ b/dojo/tools/qualys_infrascan_webgui/parser.py @@ -21,76 +21,89 @@ def issue_r(raw_row, vuln, scan_date): issue_row = {} # IP ADDRESS - issue_row['ip_address'] = raw_row.get('value') + issue_row["ip_address"] = raw_row.get("value") # FQDN - issue_row['fqdn'] = raw_row.get('name') - if issue_row['fqdn'] == "No registered hostname": - issue_row['fqdn'] = None + issue_row["fqdn"] = raw_row.get("name") + if issue_row["fqdn"] == "No registered hostname": + issue_row["fqdn"] = None # port - _port = raw_row.get('port') + _port = raw_row.get("port") # Create Endpoint - if issue_row['fqdn']: - ep = Endpoint(host=issue_row['fqdn']) + if issue_row["fqdn"]: + ep = Endpoint(host=issue_row["fqdn"]) else: - ep = Endpoint(host=issue_row['ip_address']) + ep = Endpoint(host=issue_row["ip_address"]) # OS NAME - issue_row['os'] = raw_row.findtext('OS') + issue_row["os"] = raw_row.findtext("OS") - # Scan details - VULNS//VULN indicates we only care about confirmed vulnerabilities - for vuln_cat in raw_row.findall('VULNS/CAT'): - _category = str(vuln_cat.get('value')) - for vuln_details in vuln_cat.findall('VULN'): + # Scan details - VULNS//VULN indicates we only care about confirmed + # vulnerabilities + for vuln_cat in raw_row.findall("VULNS/CAT"): + _category = str(vuln_cat.get("value")) + for vuln_details in vuln_cat.findall("VULN"): _temp = issue_row - _gid = vuln_details.get('number') + _gid = vuln_details.get("number") - _temp['port_status'] = _port + _temp["port_status"] = _port - _result = str(vuln_details.findtext('RESULT')) + _result = str(vuln_details.findtext("RESULT")) # Vuln name - _temp['vuln_name'] = vuln_details.findtext('TITLE') + _temp["vuln_name"] = vuln_details.findtext("TITLE") # Vuln Description - _description = str(vuln_details.findtext('DIAGNOSIS')) + _description = str(vuln_details.findtext("DIAGNOSIS")) # Solution Strips Heading Workaround(s) - _temp['solution'] = htmltext(str(vuln_details.findtext('SOLUTION'))) + _temp["solution"] = htmltext( + str(vuln_details.findtext("SOLUTION")) + ) # Vuln_description - _temp['vuln_description'] = "\n".join([htmltext(_description), - htmltext("**Category:** " + _category), - htmltext("**QID:** " + str(_gid)), - htmltext("**Port:** " + str(_port)), - htmltext("**Result Evidence:** " + _result), - ]) + _temp["vuln_description"] = "\n".join( + [ + htmltext(_description), + htmltext("**Category:** " + _category), + htmltext("**QID:** " + str(_gid)), + htmltext("**Port:** " + str(_port)), + htmltext("**Result Evidence:** " + _result), + ] + ) # Impact description - _temp['IMPACT'] = htmltext(str(vuln_details.findtext('CONSEQUENCE'))) + _temp["IMPACT"] = htmltext( + str(vuln_details.findtext("CONSEQUENCE")) + ) # CVE and LINKS _cl = [] - _temp_cve_details = vuln_details.iterfind('CVE_ID_LIST/CVE_ID') + _temp_cve_details = vuln_details.iterfind("CVE_ID_LIST/CVE_ID") if _temp_cve_details: - _cl = {cve_detail.findtext('ID'): cve_detail.findtext('URL') for cve_detail in _temp_cve_details} - _temp['cve'] = "\n".join(list(_cl.keys())) - _temp['links'] = "\n".join(list(_cl.values())) + _cl = { + cve_detail.findtext("ID"): cve_detail.findtext("URL") + for cve_detail in _temp_cve_details + } + _temp["cve"] = "\n".join(list(_cl.keys())) + _temp["links"] = "\n".join(list(_cl.values())) # The CVE in Qualys report might not have a CVSS score, so findings are informational by default - # unless we can find map to a Severity OR a CVSS score from the findings detail. - sev = qualys_convert_severity(vuln_details.get('severity')) + # unless we can find map to a Severity OR a CVSS score from the + # findings detail. + sev = qualys_convert_severity(vuln_details.get("severity")) refs = "\n".join(list(_cl.values())) - finding = Finding(title=_temp['vuln_name'], - mitigation=_temp['solution'], - description=_temp['vuln_description'], - severity=sev, - references=refs, - impact=_temp['IMPACT'], - vuln_id_from_tool=_gid, - date=scan_date, - ) + finding = Finding( + title=_temp["vuln_name"], + mitigation=_temp["solution"], + description=_temp["vuln_description"], + severity=sev, + references=refs, + impact=_temp["IMPACT"], + vuln_id_from_tool=_gid, + date=scan_date, + ) finding.unsaved_endpoints = list() finding.unsaved_endpoints.append(ep) ret_rows.append(finding) @@ -99,22 +112,21 @@ def issue_r(raw_row, vuln, scan_date): def qualys_convert_severity(raw_val): val = str(raw_val).strip() - if '1' == val: - return 'Info' - elif '2' == val: - return 'Low' - elif '3' == val: - return 'Medium' - elif '4' == val: - return 'High' - elif '5' == val: - return 'Critical' + if "1" == val: + return "Info" + elif "2" == val: + return "Low" + elif "3" == val: + return "Medium" + elif "4" == val: + return "High" + elif "5" == val: + return "Critical" else: - return 'Info' + return "Info" class QualysInfrascanWebguiParser(object): - def get_scan_types(self): return ["Qualys Infrastructure Scan (WebGUI XML)"] @@ -129,11 +141,11 @@ def get_findings(self, file, test): # fetch scan date e.g.: 2020-01-30T09:45:41Z scan_date = datetime.now() - for i in data.findall('HEADER/KEY'): - if i.get('value') == 'DATE': + for i in data.findall("HEADER/KEY"): + if i.get("value") == "DATE": scan_date = parser.isoparse(i.text) master_list = [] - for issue in data.findall('IP'): + for issue in data.findall("IP"): master_list += issue_r(issue, data, scan_date) return master_list From 1a382cc01568a5b1e1a4954a6734b6526656fff8 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:21:37 -0500 Subject: [PATCH 24/29] Update files in folder dojo/tools/qualys_webapp with PEP8 standars. --- dojo/tools/qualys_webapp/parser.py | 304 +++++++++++++++++++---------- 1 file changed, 203 insertions(+), 101 deletions(-) diff --git a/dojo/tools/qualys_webapp/parser.py b/dojo/tools/qualys_webapp/parser.py index eca2abc335e..48b3b52dfcb 100644 --- a/dojo/tools/qualys_webapp/parser.py +++ b/dojo/tools/qualys_webapp/parser.py @@ -24,16 +24,12 @@ # Since Info findings are not recroded in the Confirmed Vulnerability or # Potential Vulnerability categories, a severity of 1 is shown as low # in the portal. -SEVERITY_MATCH = ['Low', - 'Low', - 'Medium', - 'High', - 'Critical'] +SEVERITY_MATCH = ["Low", "Low", "Medium", "High", "Critical"] def truncate_str(value: str, maxlen: int): if len(value) > maxlen: - return value[:maxlen - 12] + " (truncated)" + return value[: maxlen - 12] + " (truncated)" return value @@ -46,7 +42,19 @@ def get_cwe(cwe): return 0 -def attach_unique_extras(endpoints, requests, responses, finding, date, qid, param, payload, unique_id, active_text, test): +def attach_unique_extras( + endpoints, + requests, + responses, + finding, + date, + qid, + param, + payload, + unique_id, + active_text, + test, +): # finding should always be none, since unique ID's are being used if finding is None: finding = Finding() @@ -73,24 +81,32 @@ def attach_unique_extras(endpoints, requests, responses, finding, date, qid, par port = "" # Set port to empty string by default # Split the returned network address into host and try: # If there is port number attached to host address - host, port = parsedUrl.netloc.split(':') - except: # there's no port attached to address + host, port = parsedUrl.netloc.split(":") + except BaseException: # there's no port attached to address host = parsedUrl.netloc - finding.unsaved_endpoints.append(Endpoint( - host=truncate_str(host, 500), port=port, - path=truncate_str(path, 500), - protocol=protocol, - query=truncate_str(query, 1000), fragment=truncate_str(fragment, 500))) + finding.unsaved_endpoints.append( + Endpoint( + host=truncate_str(host, 500), + port=port, + path=truncate_str(path, 500), + protocol=protocol, + query=truncate_str(query, 1000), + fragment=truncate_str(fragment, 500), + ) + ) for i in range(0, len(requests)): - if requests[i] != '' or responses[i] != '': - finding.unsaved_req_resp.append({"req": requests[i], "resp": responses[i]}) + if requests[i] != "" or responses[i] != "": + finding.unsaved_req_resp.append( + {"req": requests[i], "resp": responses[i]} + ) if active_text is not None: - if 'fixed' in active_text.lower(): + if "fixed" in active_text.lower(): finding.active = False - # TODO: may need to look up by finding ID and mark current finding as fixed + # TODO: may need to look up by finding ID and mark current finding + # as fixed else: finding.active = True @@ -118,8 +134,10 @@ def attach_extras(endpoints, requests, responses, finding, date, qid, test): finding.unsaved_endpoints.append(Endpoint.from_uri(endpoint)) for i in range(0, len(requests)): - if requests[i] != '' or responses[i] != '': - finding.unsaved_req_resp.append({"req": requests[i], "resp": responses[i]}) + if requests[i] != "" or responses[i] != "": + finding.unsaved_req_resp.append( + {"req": requests[i], "resp": responses[i]} + ) return finding @@ -128,23 +146,23 @@ def attach_extras(endpoints, requests, responses, finding, date, qid, test): # found in the this section of the report def get_request(request): if request is not None: - header = '' - header += str(request.findtext('METHOD')) + ': ' - header += str(request.findtext('URL')) + '\n' - headers = request.find('HEADERS') + header = "" + header += str(request.findtext("METHOD")) + ": " + header += str(request.findtext("URL")) + "\n" + headers = request.find("HEADERS") if headers is not None: - for head in headers.iter('HEADER'): - header += str(head.findtext('key')) + ': ' - header += str(head.findtext('value')) + '\n' + for head in headers.iter("HEADER"): + header += str(head.findtext("key")) + ": " + header += str(head.findtext("value")) + "\n" return str(header) - return '' + return "" # Build a response string def get_response(response): if response is not None: - return decode_tag(response.find('CONTENTS')) - return '' + return decode_tag(response.find("CONTENTS")) + return "" # Decode an XML tag with base64 if the tag has base64=true set. @@ -162,127 +180,152 @@ def decode_tag(tag): def get_request_response(payloads): requests = [] responses = [] - for payload in payloads.iter('PAYLOAD'): - requests.append(get_request(payload.find('REQUEST'))) - responses.append(get_response(payload.find('RESPONSE'))) + for payload in payloads.iter("PAYLOAD"): + requests.append(get_request(payload.find("REQUEST"))) + responses.append(get_response(payload.find("RESPONSE"))) return [requests, responses] -def get_unique_vulnerabilities(vulnerabilities, test, is_info=False, is_app_report=False): +def get_unique_vulnerabilities( + vulnerabilities, test, is_info=False, is_app_report=False +): findings = {} # Iterate through all vulnerabilites to pull necessary info for vuln in vulnerabilities: urls = [] - requests = response = '' - qid = int(vuln.findtext('QID')) - url = vuln.findtext('URL') + requests = response = "" + qid = int(vuln.findtext("QID")) + url = vuln.findtext("URL") if url is not None: urls.append(str(url)) - access_path = vuln.find('ACCESS_PATH') + access_path = vuln.find("ACCESS_PATH") if access_path is not None: - urls += [url.text for url in access_path.iter('URL')] - payloads = vuln.find('PAYLOADS') + urls += [url.text for url in access_path.iter("URL")] + payloads = vuln.find("PAYLOADS") if payloads is not None: req_resps = get_request_response(payloads) else: req_resps = [[], []] if is_info: - raw_finding_date = vuln.findtext('LAST_TIME_DETECTED') + raw_finding_date = vuln.findtext("LAST_TIME_DETECTED") elif is_app_report: - raw_finding_date = vuln.findtext('FIRST_TIME_DETECTED') + raw_finding_date = vuln.findtext("FIRST_TIME_DETECTED") else: - raw_finding_date = vuln.findtext('DETECTION_DATE') + raw_finding_date = vuln.findtext("DETECTION_DATE") # Qualys uses a non-standard date format. if raw_finding_date is not None: if raw_finding_date.endswith("GMT"): - finding_date = datetime.strptime(raw_finding_date, "%d %b %Y %I:%M%p GMT") + finding_date = datetime.strptime( + raw_finding_date, "%d %b %Y %I:%M%p GMT" + ) else: - finding_date = datetime.strptime(raw_finding_date, "%d %b %Y %I:%M%p GMT%z") + finding_date = datetime.strptime( + raw_finding_date, "%d %b %Y %I:%M%p GMT%z" + ) else: finding_date = None # Updating to include customized values - unique_id = vuln.findtext('UNIQUE_ID') - active_text = vuln.findtext('STATUS') + unique_id = vuln.findtext("UNIQUE_ID") + active_text = vuln.findtext("STATUS") param = None payload = None if not is_info: - param = vuln.findtext('PARAM') - payload = vuln.findtext('PAYLOADS/PAYLOAD/PAYLOAD') - - findings[unique_id] = attach_unique_extras(urls, req_resps[0], req_resps[1], None, finding_date, qid, param, payload, - unique_id, active_text, test) + param = vuln.findtext("PARAM") + payload = vuln.findtext("PAYLOADS/PAYLOAD/PAYLOAD") + + findings[unique_id] = attach_unique_extras( + urls, + req_resps[0], + req_resps[1], + None, + finding_date, + qid, + param, + payload, + unique_id, + active_text, + test, + ) return findings # Traverse and retreive any information in the VULNERABILITY_LIST # section of the report. This includes all endpoints and request/response pairs -def get_vulnerabilities(vulnerabilities, test, is_info=False, is_app_report=False): +def get_vulnerabilities( + vulnerabilities, test, is_info=False, is_app_report=False +): findings = {} # Iterate through all vulnerabilites to pull necessary info for vuln in vulnerabilities: urls = [] - requests = response = '' - qid = int(vuln.findtext('QID')) - url = vuln.findtext('URL') + requests = response = "" + qid = int(vuln.findtext("QID")) + url = vuln.findtext("URL") if url is not None: urls.append(str(url)) - access_path = vuln.find('ACCESS_PATH') + access_path = vuln.find("ACCESS_PATH") if access_path is not None: - urls += [url.text for url in access_path.iter('URL')] - payloads = vuln.find('PAYLOADS') + urls += [url.text for url in access_path.iter("URL")] + payloads = vuln.find("PAYLOADS") if payloads is not None: req_resps = get_request_response(payloads) else: req_resps = [[], []] if is_info: - raw_finding_date = vuln.findtext('LAST_TIME_DETECTED') + raw_finding_date = vuln.findtext("LAST_TIME_DETECTED") elif is_app_report: - raw_finding_date = vuln.findtext('FIRST_TIME_DETECTED') + raw_finding_date = vuln.findtext("FIRST_TIME_DETECTED") else: - raw_finding_date = vuln.findtext('DETECTION_DATE') + raw_finding_date = vuln.findtext("DETECTION_DATE") # Qualys uses a non-standard date format. if raw_finding_date is not None: if raw_finding_date.endswith("GMT"): - finding_date = datetime.strptime(raw_finding_date, "%d %b %Y %I:%M%p GMT") + finding_date = datetime.strptime( + raw_finding_date, "%d %b %Y %I:%M%p GMT" + ) else: - finding_date = datetime.strptime(raw_finding_date, "%d %b %Y %I:%M%p GMT%z") + finding_date = datetime.strptime( + raw_finding_date, "%d %b %Y %I:%M%p GMT%z" + ) else: finding_date = None finding = findings.get(qid, None) - findings[qid] = attach_extras(urls, req_resps[0], req_resps[1], finding, finding_date, qid, test) + findings[qid] = attach_extras( + urls, req_resps[0], req_resps[1], finding, finding_date, qid, test + ) return findings # Retrieve information from a single glossary entry such as description, # severity, title, impact, mitigation, and CWE def get_glossary_item(glossary, finding, is_info=False, enable_weakness=False): - title = glossary.findtext('TITLE') + title = glossary.findtext("TITLE") if title is not None: finding.title = str(title) - severity = glossary.findtext('SEVERITY') + severity = glossary.findtext("SEVERITY") if severity is not None: - group = glossary.findtext('GROUP') + group = glossary.findtext("GROUP") if is_info and (not enable_weakness or group in ("DIAG", "IG")): # Scan Diagnostics are always Info. finding.severity = "Info" else: finding.severity = SEVERITY_MATCH[int(severity) - 1] - description = glossary.findtext('DESCRIPTION') + description = glossary.findtext("DESCRIPTION") if description is not None: finding.description = str(description) - impact = glossary.findtext('IMPACT') + impact = glossary.findtext("IMPACT") if impact is not None: finding.impact = str(impact) - solution = glossary.findtext('SOLUTION') + solution = glossary.findtext("SOLUTION") if solution is not None: finding.mitigation = str(solution) - cwe = glossary.findtext('CWE') + cwe = glossary.findtext("CWE") if cwe is not None: finding.cwe = int(get_cwe(str(cwe))) return finding @@ -290,30 +333,44 @@ def get_glossary_item(glossary, finding, is_info=False, enable_weakness=False): # Retrieve information from a single information gathered entry def get_info_item(info_gathered, finding): - data = info_gathered.find('DATA') + data = info_gathered.find("DATA") if data is not None: - finding.description += '\n\n' + decode_tag(data) + finding.description += "\n\n" + decode_tag(data) return finding # Create findings report for all unique vulnerabilities in the report -def get_unique_items(vulnerabilities, info_gathered, glossary, is_app_report, test, enable_weakness=False): - ig_qid_list = [int(ig.findtext('QID')) for ig in info_gathered] - g_qid_list = [int(g.findtext('QID')) for g in glossary] +def get_unique_items( + vulnerabilities, + info_gathered, + glossary, + is_app_report, + test, + enable_weakness=False, +): + ig_qid_list = [int(ig.findtext("QID")) for ig in info_gathered] + g_qid_list = [int(g.findtext("QID")) for g in glossary] # This dict has findings mapped by unique ID to remove any duplicates findings = {} - total = 0 - for unique_id, finding in get_unique_vulnerabilities(vulnerabilities, test, False, is_app_report).items(): + for unique_id, finding in get_unique_vulnerabilities( + vulnerabilities, test, False, is_app_report + ).items(): qid = int(finding.vuln_id_from_tool) if qid in g_qid_list: index = g_qid_list.index(qid) - findings[unique_id] = get_glossary_item(glossary[index], finding, enable_weakness) - for unique_id, finding in get_unique_vulnerabilities(info_gathered, test, True, is_app_report).items(): + findings[unique_id] = get_glossary_item( + glossary[index], finding, enable_weakness + ) + for unique_id, finding in get_unique_vulnerabilities( + info_gathered, test, True, is_app_report + ).items(): qid = int(finding.vuln_id_from_tool) if qid in g_qid_list: index = g_qid_list.index(qid) - finding = get_glossary_item(glossary[index], finding, True, enable_weakness) + finding = get_glossary_item( + glossary[index], finding, True, enable_weakness + ) if qid in ig_qid_list: index = ig_qid_list.index(qid) findings[unique_id] = get_info_item(info_gathered[index], finding) @@ -321,21 +378,36 @@ def get_unique_items(vulnerabilities, info_gathered, glossary, is_app_report, te # Create finding items for all vulnerabilities in the report -def get_items(vulnerabilities, info_gathered, glossary, is_app_report, test, enable_weakness=False): - ig_qid_list = [int(ig.findtext('QID')) for ig in info_gathered] - g_qid_list = [int(g.findtext('QID')) for g in glossary] +def get_items( + vulnerabilities, + info_gathered, + glossary, + is_app_report, + test, + enable_weakness=False, +): + ig_qid_list = [int(ig.findtext("QID")) for ig in info_gathered] + g_qid_list = [int(g.findtext("QID")) for g in glossary] # This dict has findings mapped by QID to remove any duplicates findings = {} - for qid, finding in get_vulnerabilities(vulnerabilities, test, False, is_app_report).items(): + for qid, finding in get_vulnerabilities( + vulnerabilities, test, False, is_app_report + ).items(): if qid in g_qid_list: index = g_qid_list.index(qid) - findings[qid] = get_glossary_item(glossary[index], finding, enable_weakness) - for qid, finding in get_vulnerabilities(info_gathered, test, True, is_app_report).items(): + findings[qid] = get_glossary_item( + glossary[index], finding, enable_weakness + ) + for qid, finding in get_vulnerabilities( + info_gathered, test, True, is_app_report + ).items(): if qid in g_qid_list: index = g_qid_list.index(qid) - finding = get_glossary_item(glossary[index], finding, True, enable_weakness) + finding = get_glossary_item( + glossary[index], finding, True, enable_weakness + ) if qid in ig_qid_list: index = ig_qid_list.index(qid) findings[qid] = get_info_item(info_gathered[index], finding) @@ -347,28 +419,54 @@ def qualys_webapp_parser(qualys_xml_file, test, unique, enable_weakness=False): if qualys_xml_file is None: return [] - # supposed to be safe against XEE: https://docs.python.org/3/library/xml.html#xml-vulnerabilities + # supposed to be safe against XEE: + # https://docs.python.org/3/library/xml.html#xml-vulnerabilities tree = xml.etree.ElementTree.parse(qualys_xml_file) - is_app_report = tree.getroot().tag == 'WAS_WEBAPP_REPORT' + is_app_report = tree.getroot().tag == "WAS_WEBAPP_REPORT" if is_app_report: - vulnerabilities = tree.findall('./RESULTS/WEB_APPLICATION/VULNERABILITY_LIST/VULNERABILITY') - info_gathered = tree.findall('./RESULTS/WEB_APPLICATION/INFORMATION_GATHERED_LIST/INFORMATION_GATHERED') + vulnerabilities = tree.findall( + "./RESULTS/WEB_APPLICATION/VULNERABILITY_LIST/VULNERABILITY" + ) + info_gathered = tree.findall( + "./RESULTS/WEB_APPLICATION/INFORMATION_GATHERED_LIST/INFORMATION_GATHERED" + ) else: - vulnerabilities = tree.findall('./RESULTS/VULNERABILITY_LIST/VULNERABILITY') - info_gathered = tree.findall('./RESULTS/INFORMATION_GATHERED_LIST/INFORMATION_GATHERED') - glossary = tree.findall('./GLOSSARY/QID_LIST/QID') + vulnerabilities = tree.findall( + "./RESULTS/VULNERABILITY_LIST/VULNERABILITY" + ) + info_gathered = tree.findall( + "./RESULTS/INFORMATION_GATHERED_LIST/INFORMATION_GATHERED" + ) + glossary = tree.findall("./GLOSSARY/QID_LIST/QID") if unique: - items = list(get_unique_items(vulnerabilities, info_gathered, glossary, is_app_report, test, enable_weakness).values()) + items = list( + get_unique_items( + vulnerabilities, + info_gathered, + glossary, + is_app_report, + test, + enable_weakness, + ).values() + ) else: - items = list(get_items(vulnerabilities, info_gathered, glossary, is_app_report, test, enable_weakness).values()) + items = list( + get_items( + vulnerabilities, + info_gathered, + glossary, + is_app_report, + test, + enable_weakness, + ).values() + ) return items class QualysWebAppParser(object): - def get_scan_types(self): return ["Qualys Webapp Scan"] @@ -378,5 +476,9 @@ def get_label_for_scan_types(self, scan_type): def get_description_for_scan_types(self, scan_type): return "Qualys WebScan output files can be imported in XML format." - def get_findings(self, file, test, enable_weakness=QUALYS_WAS_WEAKNESS_IS_VULN): - return qualys_webapp_parser(file, test, QUALYS_WAS_UNIQUE_ID, enable_weakness) + def get_findings( + self, file, test, enable_weakness=QUALYS_WAS_WEAKNESS_IS_VULN + ): + return qualys_webapp_parser( + file, test, QUALYS_WAS_UNIQUE_ID, enable_weakness + ) From 1a5af8fa541affb0226be596b217cc5f893f63e6 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:23:21 -0500 Subject: [PATCH 25/29] Update files in folder dojo/tools/retirejs with PEP8 standars. --- dojo/tools/retirejs/parser.py | 57 +++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/dojo/tools/retirejs/parser.py b/dojo/tools/retirejs/parser.py index 2ddbe7e523f..2482d517dc2 100644 --- a/dojo/tools/retirejs/parser.py +++ b/dojo/tools/retirejs/parser.py @@ -5,7 +5,6 @@ class RetireJsParser(object): - def get_scan_types(self): return ["Retire.js Scan"] @@ -21,44 +20,56 @@ def get_findings(self, json_output, test): def get_items(self, tree, test): items = {} - if 'data' in tree: - tree = tree['data'] + if "data" in tree: + tree = tree["data"] for node in tree: - for result in node['results']: - if 'vulnerabilities' in result: - for vulnerability in result['vulnerabilities']: - item = self.get_item(vulnerability, test, node['file']) - item.title += " (" + result['component'] + ", " + result['version'] + ")" - item.description += "\n\n Raw Result: " + str(json.dumps(vulnerability, indent=4, sort_keys=True)) + for result in node["results"]: + if "vulnerabilities" in result: + for vulnerability in result["vulnerabilities"]: + item = self.get_item(vulnerability, test, node["file"]) + item.title += ( + " (" + + result["component"] + + ", " + + result["version"] + + ")" + ) + item.description += "\n\n Raw Result: " + str( + json.dumps(vulnerability, indent=4, sort_keys=True) + ) item.references = item.references - item.component_name = result.get('component') - item.component_version = result.get('version') - item.file_path = node['file'] + item.component_name = result.get("component") + item.component_version = result.get("version") + item.file_path = node["file"] - encrypted_file = node['file'] - unique_key = hashlib.md5((item.title + item.references + encrypted_file).encode()).hexdigest() + encrypted_file = node["file"] + unique_key = hashlib.md5( + ( + item.title + item.references + encrypted_file + ).encode() + ).hexdigest() items[unique_key] = item return list(items.values()) def get_item(self, item_node, test, file): title = "" - if 'identifiers' in item_node: - if 'summary' in item_node['identifiers']: - title = item_node['identifiers']['summary'] - elif 'CVE' in item_node['identifiers']: - title = "".join(item_node['identifiers']['CVE']) - elif 'osvdb' in item_node['identifiers']: - title = "".join(item_node['identifiers']['osvdb']) + if "identifiers" in item_node: + if "summary" in item_node["identifiers"]: + title = item_node["identifiers"]["summary"] + elif "CVE" in item_node["identifiers"]: + title = "".join(item_node["identifiers"]["CVE"]) + elif "osvdb" in item_node["identifiers"]: + title = "".join(item_node["identifiers"]["osvdb"]) finding = Finding( title=title, test=test, cwe=1035, # Vulnerable Third Party Component - severity=item_node['severity'].title(), + severity=item_node["severity"].title(), description=title + "\n\n Affected File - " + file, file_path=file, - references="\n".join(item_node['info']), + references="\n".join(item_node["info"]), false_p=False, duplicate=False, out_of_scope=False, From d1d4f5f96ae68a14916cccaad735e111951773b1 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:24:16 -0500 Subject: [PATCH 26/29] Update files in folder dojo/tools/risk_recon with PEP8 standars. --- dojo/tools/risk_recon/api.py | 55 ++++++++++----------- dojo/tools/risk_recon/parser.py | 84 ++++++++++++++++++++++----------- 2 files changed, 84 insertions(+), 55 deletions(-) diff --git a/dojo/tools/risk_recon/api.py b/dojo/tools/risk_recon/api.py index dc420067a31..0ac61f805d5 100644 --- a/dojo/tools/risk_recon/api.py +++ b/dojo/tools/risk_recon/api.py @@ -11,17 +11,17 @@ def __init__(self, api_key, endpoint, data): if not self.key: raise Exception( - 'Please supply a Risk Recon API key. \n' - 'This can be generated in the system admin panel. \n' - 'See https://documentation.defectdojo.com/integrations/import/#risk-recon-api-importer \n' + "Please supply a Risk Recon API key. \n" + "This can be generated in the system admin panel. \n" + "See https://documentation.defectdojo.com/integrations/import/#risk-recon-api-importer \n" ) if not self.url: raise Exception( - 'Please supply a Risk Recon API url. \n' - 'A general url is https://api.riskrecon.com/v1/ \n' - 'See https://documentation.defectdojo.com/integrations/import/#risk-recon-api-importer \n' + "Please supply a Risk Recon API url. \n" + "A general url is https://api.riskrecon.com/v1/ \n" + "See https://documentation.defectdojo.com/integrations/import/#risk-recon-api-importer \n" ) - if self.url.endswith('/'): + if self.url.endswith("/"): self.url = endpoint[:-1] self.session = requests.Session() self.map_toes() @@ -29,11 +29,8 @@ def __init__(self, api_key, endpoint, data): def map_toes(self): response = self.session.get( - url='{}/toes'.format(self.url), - headers={ - 'accept': 'application/json', - 'Authorization': self.key - } + url="{}/toes".format(self.url), + headers={"accept": "application/json", "Authorization": self.key}, ) if response.ok: @@ -41,24 +38,26 @@ def map_toes(self): data = response.json() if isinstance(self.data, list): for company in self.data: - name = company.get('name', None) - filters = company.get('filters', None) + name = company.get("name", None) + filters = company.get("filters", None) if name: comps[name] = filters name_list = comps.keys() for item in data: - toe_id = item.get('toe_id', None) - name = item.get('toe_short_name', None) + toe_id = item.get("toe_id", None) + name = item.get("toe_short_name", None) if not comps or name in name_list: filters = comps.get(name, None) self.toe_map[toe_id] = filters if filters else self.data else: - raise Exception('Unable to query Target of Evaluations due to {} - {}'.format( - response.status_code, response.content - )) + raise Exception( + "Unable to query Target of Evaluations due to {} - {}".format( + response.status_code, response.content + ) + ) def filter_finding(self, finding): - filters = self.toe_map[finding['toe_id']] + filters = self.toe_map[finding["toe_id"]] if not filters: return False @@ -72,11 +71,11 @@ def filter_finding(self, finding): def get_findings(self): for toe in self.toe_map.keys(): response = self.session.get( - url='{}/findings/{}'.format(self.url, toe), + url="{}/findings/{}".format(self.url, toe), headers={ - 'accept': 'application/json', - 'Authorization': self.key - } + "accept": "application/json", + "Authorization": self.key, + }, ) if response.ok: @@ -85,6 +84,8 @@ def get_findings(self): if not self.filter_finding(finding): self.findings.append(finding) else: - raise Exception('Unable to collect findings from toe: {} due to {} - {}'.format( - toe, response.status_code, response.content - )) + raise Exception( + "Unable to collect findings from toe: {} due to {} - {}".format( + toe, response.status_code, response.content + ) + ) diff --git a/dojo/tools/risk_recon/parser.py b/dojo/tools/risk_recon/parser.py index 7c37c8bb5e2..ca9d5a5d2f1 100644 --- a/dojo/tools/risk_recon/parser.py +++ b/dojo/tools/risk_recon/parser.py @@ -6,7 +6,6 @@ class RiskReconParser(object): - def get_scan_types(self): return ["Risk Recon API Importer"] @@ -20,48 +19,75 @@ def get_findings(self, filename, test): if filename: tree = filename.read() try: - data = json.loads(str(tree, 'utf-8')) - except: + data = json.loads(str(tree, "utf-8")) + except BaseException: data = json.loads(tree) findings = [] - if not data.get('test', None): + if not data.get("test", None): api = RiskReconAPI( - data.get('api_key', None), - data.get('url_endpoint', None), - data.get('companies', data.get('filters', [])), + data.get("api_key", None), + data.get("url_endpoint", None), + data.get("companies", data.get("filters", [])), ) findings = api.findings else: - findings = data.get('findings') + findings = data.get("findings") return self._get_findings_internal(findings, test) def _get_findings_internal(self, findings, test): dupes = dict() for item in findings: - findingdetail = '' - title = item.get('vendor') + ': ' + item.get('finding') + ' - ' + item.get('domain_name') + '(' + item.get('ip_address') + ')' + findingdetail = "" + title = ( + item.get("vendor") + + ": " + + item.get("finding") + + " - " + + item.get("domain_name") + + "(" + + item.get("ip_address") + + ")" + ) # Finding details information - findingdetail += '**ID:** ' + item.get('finding_id') + '\n' - findingdetail += '**Context:** ' + item.get('finding_context') + '\n' - findingdetail += '**Value:** ' + item.get('finding_data_value') + '\n' - findingdetail += '**Hosting Provider:** ' + item.get('hosting_provider') + '\n' - findingdetail += '**Host Name:** ' + item.get('host_name') + '\n' - findingdetail += '**Security Domain:** ' + item.get('security_domain') + '\n' - findingdetail += '**Security Criteria:** ' + item.get('security_criteria') + '\n' - findingdetail += '**Asset Value:** ' + item.get('asset_value') + '\n' - findingdetail += '**Country:** ' + item.get('country_name') + '\n' - findingdetail += '**Priority:** ' + item.get('priority') + '\n' - findingdetail += '**First Seen:** ' + item.get('first_seen') + '\n' - - date = dateutil.parser.parse(item.get('first_seen')) - - sev = item.get('severity', "").capitalize() + findingdetail += "**ID:** " + item.get("finding_id") + "\n" + findingdetail += ( + "**Context:** " + item.get("finding_context") + "\n" + ) + findingdetail += ( + "**Value:** " + item.get("finding_data_value") + "\n" + ) + findingdetail += ( + "**Hosting Provider:** " + item.get("hosting_provider") + "\n" + ) + findingdetail += "**Host Name:** " + item.get("host_name") + "\n" + findingdetail += ( + "**Security Domain:** " + item.get("security_domain") + "\n" + ) + findingdetail += ( + "**Security Criteria:** " + + item.get("security_criteria") + + "\n" + ) + findingdetail += ( + "**Asset Value:** " + item.get("asset_value") + "\n" + ) + findingdetail += "**Country:** " + item.get("country_name") + "\n" + findingdetail += "**Priority:** " + item.get("priority") + "\n" + findingdetail += "**First Seen:** " + item.get("first_seen") + "\n" + + date = dateutil.parser.parse(item.get("first_seen")) + + sev = item.get("severity", "").capitalize() sev = "Info" if not sev else sev - tags = item.get('security_domain')[:20] + ', ' + item.get('security_criteria')[:20] + tags = ( + item.get("security_domain")[:20] + + ", " + + item.get("security_criteria")[:20] + ) finding = Finding( title=title, @@ -71,12 +97,14 @@ def _get_findings_internal(self, findings, test): static_finding=False, dynamic_finding=True, date=date, - unique_id_from_tool=item.get('finding_id'), + unique_id_from_tool=item.get("finding_id"), nb_occurences=1, # there is no de-duplication ) finding.unsaved_tags = tags - dupe_key = item.get('finding_id', title + '|' + tags + '|' + findingdetail) + dupe_key = item.get( + "finding_id", title + "|" + tags + "|" + findingdetail + ) if dupe_key in dupes: find = dupes[dupe_key] From 0264ba49c0bad448e0ae717782a33ba6aff702bd Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:25:17 -0500 Subject: [PATCH 27/29] Update files in folder dojo/tools/rubocop with PEP8 standars. --- dojo/tools/rubocop/parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dojo/tools/rubocop/parser.py b/dojo/tools/rubocop/parser.py index 99919ebd130..db18a4619b3 100644 --- a/dojo/tools/rubocop/parser.py +++ b/dojo/tools/rubocop/parser.py @@ -4,7 +4,6 @@ class RubocopParser: - ID = "Rubocop Scan" # possible values are: From ca9d0a713cfe3772f783c01ae2a434201157e8ac Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Mon, 26 Jun 2023 01:26:19 -0500 Subject: [PATCH 28/29] Update files in folder dojo/tools/rusty_hog with PEP8 standars. --- dojo/tools/rusty_hog/parser.py | 180 +++++++++++++++++++++------------ 1 file changed, 118 insertions(+), 62 deletions(-) diff --git a/dojo/tools/rusty_hog/parser.py b/dojo/tools/rusty_hog/parser.py index 165110214af..da0baa6c83e 100644 --- a/dojo/tools/rusty_hog/parser.py +++ b/dojo/tools/rusty_hog/parser.py @@ -4,7 +4,6 @@ class RustyhogParser(object): - def get_scan_types(self): return ["Rusty Hog Scan"] @@ -24,7 +23,9 @@ def parse_json(self, json_output): def get_items(self, json_output, scanner, test): items = {} - findings = self.__getitem(vulnerabilities=self.parse_json(json_output), scanner=scanner) + findings = self.__getitem( + vulnerabilities=self.parse_json(json_output), scanner=scanner + ) for finding in findings: unique_key = "Finding {}".format(finding) items[unique_key] = finding @@ -35,16 +36,22 @@ def get_tests(self, scan_type, handle): tests = list() parsername = "Rusty Hog" for node in tree: - if 'commit' in node or 'commitHash' in node or 'parent_commit_hash' in node or 'old_file_id' in node or 'new_file_id' in node: + if ( + "commit" in node + or "commitHash" in node + or "parent_commit_hash" in node + or "old_file_id" in node + or "new_file_id" in node + ): parsername = "Choctaw Hog" break - if 'linenum' in node or 'diff' in node: + if "linenum" in node or "diff" in node: parsername = "Duroc Hog" break - if 'issue_id' in node or 'location' in node: + if "issue_id" in node or "location" in node: parsername = "Gottingen Hog" break - if 'page_id' in node: + if "page_id" in node: parsername = "Essex Hog" break test = ParserTest( @@ -52,17 +59,20 @@ def get_tests(self, scan_type, handle): type=parsername, version="", ) - if parsername == "Rusty Hog": # The outputfile is empty. A subscanner can't be classified + if ( + parsername == "Rusty Hog" + ): # The outputfile is empty. A subscanner can't be classified test.description = "The exact scanner within Rusty Hog could not be determined due to missing information within the scan result." else: test.description = parsername - test.findings = self.__getitem(vulnerabilities=tree, scanner=parsername) + test.findings = self.__getitem( + vulnerabilities=tree, scanner=parsername + ) tests.append(test) return tests def __getitem(self, vulnerabilities, scanner): findings = [] - line = "" found_secret_string = "" cwe = 200 for vulnerability in vulnerabilities: @@ -70,85 +80,131 @@ def __getitem(self, vulnerabilities, scanner): break elif scanner == "Choctaw Hog": """Choctaw Hog""" - found_secret_string = vulnerability.get('stringsFound') - description = "**This string was found:** {}".format(found_secret_string) - if vulnerability.get('commit') is not None: - description += "\n**Commit message:** {}".format(vulnerability.get('commit')) - if vulnerability.get('commitHash') is not None: - description += "\n**Commit hash:** {}".format(vulnerability.get('commitHash')) - if vulnerability.get('parent_commit_hash') is not None: - description += "\n**Parent commit hash:** {}".format(vulnerability.get('parent_commit_hash')) - if vulnerability.get('old_file_id') is not None and vulnerability.get('new_file_id') is not None: - description += "\n**Old and new file IDs:** {} - {}".format( - vulnerability.get('old_file_id'), - vulnerability.get('new_file_id')) - if vulnerability.get('old_line_num') is not None and vulnerability.get('new_line_num') is not None: - description += "\n**Old and new line numbers:** {} - {}".format( - vulnerability.get('old_line_num'), - vulnerability.get('new_line_num')) + found_secret_string = vulnerability.get("stringsFound") + description = "**This string was found:** {}".format( + found_secret_string + ) + if vulnerability.get("commit") is not None: + description += "\n**Commit message:** {}".format( + vulnerability.get("commit") + ) + if vulnerability.get("commitHash") is not None: + description += "\n**Commit hash:** {}".format( + vulnerability.get("commitHash") + ) + if vulnerability.get("parent_commit_hash") is not None: + description += "\n**Parent commit hash:** {}".format( + vulnerability.get("parent_commit_hash") + ) + if ( + vulnerability.get("old_file_id") is not None + and vulnerability.get("new_file_id") is not None + ): + description += ( + "\n**Old and new file IDs:** {} - {}".format( + vulnerability.get("old_file_id"), + vulnerability.get("new_file_id"), + ) + ) + if ( + vulnerability.get("old_line_num") is not None + and vulnerability.get("new_line_num") is not None + ): + description += ( + "\n**Old and new line numbers:** {} - {}".format( + vulnerability.get("old_line_num"), + vulnerability.get("new_line_num"), + ) + ) elif scanner == "Duroc Hog": """Duroc Hog""" - found_secret_string = vulnerability.get('stringsFound') - description = "**This string was found:** {}".format(found_secret_string) - if vulnerability.get('path') is not None: - description += "\n**Path of Issue:** {}".format(vulnerability.get('path')) - if vulnerability.get('linenum') is not None: - description += "\n**Linenum of Issue:** {}".format(vulnerability.get('linenum')) - if vulnerability.get('diff') is not None: - description += "\n**Diff:** {}".format(vulnerability.get('diff')) + found_secret_string = vulnerability.get("stringsFound") + description = "**This string was found:** {}".format( + found_secret_string + ) + if vulnerability.get("path") is not None: + description += "\n**Path of Issue:** {}".format( + vulnerability.get("path") + ) + if vulnerability.get("linenum") is not None: + description += "\n**Linenum of Issue:** {}".format( + vulnerability.get("linenum") + ) + if vulnerability.get("diff") is not None: + description += "\n**Diff:** {}".format( + vulnerability.get("diff") + ) elif scanner == "Gottingen Hog": """Gottingen Hog""" - found_secret_string = vulnerability.get('stringsFound') - description = "**This string was found:** {}".format(found_secret_string) - if vulnerability.get('issue_id') is not None: - description += "\n**JIRA Issue ID:** {}".format(vulnerability.get('issue_id')) - if vulnerability.get('location') is not None: - description += "\n**JIRA location:** {}".format(vulnerability.get('location')) - if vulnerability.get('url') is not None: - description += "\n**JIRA url:** [{}]({})".format(vulnerability.get('url'), vulnerability.get('url')) + found_secret_string = vulnerability.get("stringsFound") + description = "**This string was found:** {}".format( + found_secret_string + ) + if vulnerability.get("issue_id") is not None: + description += "\n**JIRA Issue ID:** {}".format( + vulnerability.get("issue_id") + ) + if vulnerability.get("location") is not None: + description += "\n**JIRA location:** {}".format( + vulnerability.get("location") + ) + if vulnerability.get("url") is not None: + description += "\n**JIRA url:** [{}]({})".format( + vulnerability.get("url"), vulnerability.get("url") + ) elif scanner == "Essex Hog": - found_secret_string = vulnerability.get('stringsFound') - description = "**This string was found:** {}".format(found_secret_string) - if vulnerability.get('page_id') is not None: - description += "\n**Confluence URL:** [{}]({})".format(vulnerability.get('url'), vulnerability.get('url')) - description += "\n**Confluence Page ID:** {}".format(vulnerability.get('page_id')) + found_secret_string = vulnerability.get("stringsFound") + description = "**This string was found:** {}".format( + found_secret_string + ) + if vulnerability.get("page_id") is not None: + description += "\n**Confluence URL:** [{}]({})".format( + vulnerability.get("url"), vulnerability.get("url") + ) + description += "\n**Confluence Page ID:** {}".format( + vulnerability.get("page_id") + ) """General - for all Rusty Hogs""" - file_path = vulnerability.get('path') - if vulnerability.get('date') is not None: - description += "\n**Date:** {}".format(vulnerability.get('date')) + file_path = vulnerability.get("path") + if vulnerability.get("date") is not None: + description += "\n**Date:** {}".format( + vulnerability.get("date") + ) """Finding Title""" if scanner == "Choctaw Hog": title = "{} found in Git path {} ({})".format( - vulnerability.get('reason'), - vulnerability.get('path'), - vulnerability.get('commitHash')) + vulnerability.get("reason"), + vulnerability.get("path"), + vulnerability.get("commitHash"), + ) elif scanner == "Duroc Hog": title = "{} found in path {}".format( - vulnerability.get('reason'), - vulnerability.get('path')) + vulnerability.get("reason"), vulnerability.get("path") + ) elif scanner == "Gottingen Hog": title = "{} found in Jira ID {} ({})".format( - vulnerability.get('reason'), - vulnerability.get('issue_id'), - vulnerability.get('location')) + vulnerability.get("reason"), + vulnerability.get("issue_id"), + vulnerability.get("location"), + ) elif scanner == "Essex Hog": title = "{} found in Confluence Page ID {}".format( - vulnerability.get('reason'), - vulnerability.get('page_id')) + vulnerability.get("reason"), vulnerability.get("page_id") + ) # create the finding object finding = Finding( title=title, - severity='High', + severity="High", cwe=cwe, description=description, file_path=file_path, static_finding=True, dynamic_finding=False, - payload=found_secret_string + payload=found_secret_string, ) finding.description = finding.description.strip() if scanner == "Choctaw Hog": - finding.line = int(vulnerability.get('new_line_num')) + finding.line = int(vulnerability.get("new_line_num")) finding.mitigation = "Please ensure no secret material nor confidential information is kept in clear within git repositories." elif scanner == "Duroc Hog": finding.mitigation = "Please ensure no secret material nor confidential information is kept in clear within directories, files, and archives." From 5cc690eda741b1f58c6e5d39075007108035d7a7 Mon Sep 17 00:00:00 2001 From: Alejandro Tortolero Date: Tue, 27 Jun 2023 12:52:56 -0500 Subject: [PATCH 29/29] Change BaseException to Exception --- dojo/tools/netsparker/parser.py | 4 ++-- dojo/tools/neuvector/parser.py | 4 ++-- dojo/tools/neuvector_compliance/parser.py | 4 ++-- dojo/tools/npm_audit/parser.py | 4 ++-- dojo/tools/nsp/parser.py | 4 ++-- dojo/tools/ort/parser.py | 4 ++-- dojo/tools/php_security_audit_v2/parser.py | 2 +- dojo/tools/php_symfony_security_check/parser.py | 4 ++-- dojo/tools/risk_recon/parser.py | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dojo/tools/netsparker/parser.py b/dojo/tools/netsparker/parser.py index 20d3b2e70b4..9b4b2d31135 100644 --- a/dojo/tools/netsparker/parser.py +++ b/dojo/tools/netsparker/parser.py @@ -20,7 +20,7 @@ def get_findings(self, filename, test): tree = filename.read() try: data = json.loads(str(tree, "utf-8-sig")) - except BaseException: + except Exception: data = json.loads(tree) dupes = dict() scan_date = datetime.datetime.strptime( @@ -33,7 +33,7 @@ def get_findings(self, filename, test): if "Cwe" in item["Classification"]: try: cwe = int(item["Classification"]["Cwe"].split(",")[0]) - except BaseException: + except Exception: cwe = None else: cwe = None diff --git a/dojo/tools/neuvector/parser.py b/dojo/tools/neuvector/parser.py index a3817a71ed8..17be7635686 100644 --- a/dojo/tools/neuvector/parser.py +++ b/dojo/tools/neuvector/parser.py @@ -23,9 +23,9 @@ def parse_json(self, json_output): data = json_output.read() try: tree = json.loads(str(data, "utf-8")) - except BaseException: + except Exception: tree = json.loads(data) - except BaseException: + except Exception: raise ValueError("Invalid format") return tree diff --git a/dojo/tools/neuvector_compliance/parser.py b/dojo/tools/neuvector_compliance/parser.py index f77f11603e9..74e5e515fd1 100644 --- a/dojo/tools/neuvector_compliance/parser.py +++ b/dojo/tools/neuvector_compliance/parser.py @@ -20,9 +20,9 @@ def parse_json(json_output): data = json_output.read() try: tree = json.loads(str(data, "utf-8")) - except BaseException: + except Exception: tree = json.loads(data) - except BaseException: + except Exception: raise ValueError("Invalid format") return tree diff --git a/dojo/tools/npm_audit/parser.py b/dojo/tools/npm_audit/parser.py index c3f97e49134..968d00e0c9a 100644 --- a/dojo/tools/npm_audit/parser.py +++ b/dojo/tools/npm_audit/parser.py @@ -29,9 +29,9 @@ def parse_json(self, json_output): data = json_output.read() try: tree = json.loads(str(data, "utf-8")) - except BaseException: + except Exception: tree = json.loads(data) - except BaseException: + except Exception: raise ValueError("Invalid format, unable to parse json.") if tree.get("auditReportVersion"): diff --git a/dojo/tools/nsp/parser.py b/dojo/tools/nsp/parser.py index afec4b8ee93..40a7dcb66ab 100644 --- a/dojo/tools/nsp/parser.py +++ b/dojo/tools/nsp/parser.py @@ -25,9 +25,9 @@ def parse_json(self, json_output): data = json_output.read() try: tree = json.loads(str(data, "utf-8")) - except BaseException: + except Exception: tree = json.loads(data) - except BaseException: + except Exception: raise ValueError("Invalid format") return tree diff --git a/dojo/tools/ort/parser.py b/dojo/tools/ort/parser.py index da8f485b46c..d2811d3e170 100644 --- a/dojo/tools/ort/parser.py +++ b/dojo/tools/ort/parser.py @@ -32,9 +32,9 @@ def parse_json(self, json_output): data = json_output.read() try: tree = json.loads(str(data, "utf-8")) - except BaseException: + except Exception: tree = json.loads(data) - except BaseException: + except Exception: raise ValueError("Invalid format") return tree diff --git a/dojo/tools/php_security_audit_v2/parser.py b/dojo/tools/php_security_audit_v2/parser.py index e93165a14cc..f1ee8022c1a 100644 --- a/dojo/tools/php_security_audit_v2/parser.py +++ b/dojo/tools/php_security_audit_v2/parser.py @@ -18,7 +18,7 @@ def get_findings(self, filename, test): tree = filename.read() try: data = json.loads(str(tree, "utf-8")) - except BaseException: + except Exception: data = json.loads(tree) dupes = dict() diff --git a/dojo/tools/php_symfony_security_check/parser.py b/dojo/tools/php_symfony_security_check/parser.py index e00b9ac975b..c5fb5118804 100644 --- a/dojo/tools/php_symfony_security_check/parser.py +++ b/dojo/tools/php_symfony_security_check/parser.py @@ -24,9 +24,9 @@ def parse_json(self, json_file): data = json_file.read() try: tree = json.loads(str(data, "utf-8")) - except BaseException: + except Exception: tree = json.loads(data) - except BaseException: + except Exception: raise Exception("Invalid format") return tree diff --git a/dojo/tools/risk_recon/parser.py b/dojo/tools/risk_recon/parser.py index ca9d5a5d2f1..8c70496d691 100644 --- a/dojo/tools/risk_recon/parser.py +++ b/dojo/tools/risk_recon/parser.py @@ -20,7 +20,7 @@ def get_findings(self, filename, test): tree = filename.read() try: data = json.loads(str(tree, "utf-8")) - except BaseException: + except Exception: data = json.loads(tree) findings = []