✨ advance import-scan api to have an additional field to finetune vulnerability parsers

dojo/api_v2/serializers.py

@@ -2135,6 +2135,10 @@ class ImportScanSerializer(serializers.Serializer):
         help_text="If set to True, the tags will be applied to the findings",
         required=False,
     )
+    parser_custom_setting = serializers.CharField(
+        help_text="You can specify custom parser settings, but please take a look at the docs",
+        required=False,
+    )
 
     def save(self, push_to_jira=False):
         data = self.validated_data
@@ -2157,6 +2161,7 @@ def save(self, push_to_jira=False):
         source_code_management_uri = data.get(
             "source_code_management_uri", None
         )
+        parser_custom_setting = data.get("parser_custom_setting", None)
 
         if "active" in self.initial_data:
             active = data.get("active")
@@ -2247,6 +2252,7 @@ def save(self, push_to_jira=False):
                 title=test_title,
                 create_finding_groups_for_all_findings=create_finding_groups_for_all_findings,
                 apply_tags_to_findings=apply_tags_to_findings,
+                parser_custom_setting=parser_custom_setting,
             )
 
             if test:
@@ -2419,6 +2425,10 @@ class ReImportScanSerializer(TaggitSerializer, serializers.Serializer):
         help_text="If set to True, the tags will be applied to the findings",
         required=False
     )
+    parser_custom_setting = serializers.CharField(
+        help_text="You can specify custom parser settings, but please take a look at the docs",
+        required=False,
+    )
 
     def save(self, push_to_jira=False):
         logger.debug("push_to_jira: %s", push_to_jira)
@@ -2432,6 +2442,7 @@ def save(self, push_to_jira=False):
             "close_old_findings_product_scope"
         )
         apply_tags_to_findings = data.get("apply_tags_to_findings", False)
+        parser_custom_setting = data.get("parser_custom_setting", False)
         do_not_reactivate = data.get("do_not_reactivate", False)
         version = data.get("version", None)
         build_id = data.get("build_id", None)
@@ -2533,6 +2544,7 @@ def save(self, push_to_jira=False):
                     do_not_reactivate=do_not_reactivate,
                     create_finding_groups_for_all_findings=create_finding_groups_for_all_findings,
                     apply_tags_to_findings=apply_tags_to_findings,
+                    parser_custom_setting=parser_custom_setting,
                 )
 
                 if test_import:

dojo/forms.py

@@ -477,6 +477,7 @@ class ImportScanForm(forms.Form):
         required=False,
         initial=False
     )
+    parser_custom_setting = forms.CharField(max_length=100, required=False, help_text="This is a field to customize (finetune) the behavior of single parsers.")
 
     if is_finding_groups_enabled():
         group_by = forms.ChoiceField(required=False, choices=Finding_Group.GROUP_BY_OPTIONS, help_text='Choose an option to automatically group new findings by the chosen option.')

dojo/importers/importer/importer.py

@@ -244,7 +244,7 @@ def close_old_findings(self, test, scan_date_time, user, push_to_jira=None, serv
     def import_scan(self, scan, scan_type, engagement, lead, environment, active=None, verified=None, tags=None, minimum_severity=None,
                     user=None, endpoints_to_add=None, scan_date=None, version=None, branch_tag=None, build_id=None,
                     commit_hash=None, push_to_jira=None, close_old_findings=False, close_old_findings_product_scope=False,
-                    group_by=None, api_scan_configuration=None, service=None, title=None, create_finding_groups_for_all_findings=True, apply_tags_to_findings=False):
+                    group_by=None, api_scan_configuration=None, service=None, title=None, create_finding_groups_for_all_findings=True, apply_tags_to_findings=False, parser_custom_setting=None):
 
         logger.debug(f'IMPORT_SCAN: parameters: {locals()}')
 
@@ -312,7 +312,7 @@ def import_scan(self, scan, scan_type, engagement, lead, environment, active=Non
             logger.debug('IMPORT_SCAN: Parse findings')
             parser = get_parser(scan_type)
             try:
-                parsed_findings = parser.get_findings(scan, test)
+                parsed_findings = parser.get_findings(scan, test, parser_custom_setting)
             except ValueError as e:
                 logger.warning(e)
                 raise ValidationError(e)
@@ -367,7 +367,6 @@ def import_scan(self, scan, scan_type, engagement, lead, environment, active=Non
                 for finding in test_import.findings_affected.all():
                     for tag in tags:
                         finding.tags.add(tag)
-
         logger.debug('IMPORT_SCAN: Generating notifications')
         notifications_helper.notify_test_created(test)
         updated_count = len(new_findings) + len(closed_findings)

dojo/importers/reimporter/reimporter.py

@@ -572,6 +572,7 @@ def reimport_scan(
         do_not_reactivate=False,
         create_finding_groups_for_all_findings=True,
         apply_tags_to_findings=False,
+        parser_custom_setting=None,
     ):
 
         logger.debug(f"REIMPORT_SCAN: parameters: {locals()}")
@@ -607,7 +608,7 @@ def reimport_scan(
         else:
             logger.debug("REIMPORT_SCAN: Parse findings")
             try:
-                parsed_findings = parser.get_findings(scan, test)
+                parsed_findings = parser.get_findings(scan, test, parser_custom_setting)
             except ValueError as e:
                 logger.warning(e)
                 raise ValidationError(e)

dojo/tools/acunetix/parser.py

@@ -23,7 +23,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "XML format"
 
-    def get_findings(self, xml_output, test):
+    def get_findings(self, xml_output, test, parser_custom_setting=None):
         root = parse(xml_output).getroot()
 
         dupes = dict()

dojo/tools/acunetix360/parser.py

@@ -16,7 +16,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Acunetix360 JSON format."
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         data = json.load(filename)
         dupes = dict()
         scan_date = parser.parse(data["Generated"])

dojo/tools/anchore_engine/parser.py

@@ -13,7 +13,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Anchore-CLI JSON vulnerability report format."
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         data = json.load(filename)
         dupes = dict()
         for item in data["vulnerabilities"]:

dojo/tools/anchore_enterprise/parser.py

@@ -19,7 +19,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Anchore-CLI JSON policy check report format."
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         content = filename.read()
         try:
             data = json.loads(str(content, "utf-8"))

dojo/tools/anchore_grype/parser.py

@@ -23,7 +23,7 @@ def get_description_for_scan_types(self, scan_type):
             "format"
         )
 
-    def get_findings(self, file, test):
+    def get_findings(self, file, test, parser_custom_setting=None):
         data = json.load(file)
         dupes = dict()
         for item in data.get("matches", []):

dojo/tools/anchorectl_policies/parser.py

@@ -19,7 +19,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "AnchoreCTLs JSON policies report format."
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         content = filename.read()
         try:
             data = json.loads(str(content, "utf-8"))

dojo/tools/anchorectl_vulns/parser.py

@@ -13,7 +13,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "AnchoreCTLs JSON vulnerability report format."
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         data = json.load(filename)
         dupes = dict()
         for item in data:

dojo/tools/api_blackduck/importer.py

@@ -11,7 +11,7 @@ class BlackduckApiImporter(object):
 
     config_id = "BlackDuck API"
 
-    def get_findings(self, test):
+    def get_findings(self, test, parser_custom_setting=None):
         client, config = self.prepare_client(test)
         project = client.get_project_by_name(config.service_key_1)
         version = client.get_version_by_name(project, config.service_key_2)

dojo/tools/api_blackduck/parser.py

@@ -36,7 +36,7 @@ def api_scan_configuration_hint(self):
             "<b>Service key 2</b> has to be set to the version of the project"
         )
 
-    def get_findings(self, file, test):
+    def get_findings(self, file, test, parser_custom_setting=None):
         if file is None:
             data = BlackduckApiImporter().get_findings(test)
         else:

dojo/tools/api_bugcrowd/importer.py

@@ -11,7 +11,7 @@ class BugcrowdApiImporter(object):
     Import from Bugcrowd API
     """
 
-    def get_findings(self, test):
+    def get_findings(self, test, parser_custom_setting=None):
         client, config = self.prepare_client(test)
         logger.debug(
             "Fetching submissions program {} and target {}".format(

dojo/tools/api_bugcrowd/parser.py

@@ -46,7 +46,7 @@ def api_scan_configuration_hint(self):
             "if not supplied, will fetch all submissions in the program"
         )
 
-    def get_findings(self, file, test):
+    def get_findings(self, file, test, parser_custom_setting=None):
         api_scan_config = None
         if file is None:
             data, api_scan_config = BugcrowdApiImporter().get_findings(test)

dojo/tools/api_cobalt/importer.py

@@ -11,7 +11,7 @@ class CobaltApiImporter(object):
     Import from Cobalt.io API
     """
 
-    def get_findings(self, test):
+    def get_findings(self, test, parser_custom_setting=None):
         client, config = self.prepare_client(test)
         findings = client.get_findings(config.service_key_1)
         return findings

dojo/tools/api_cobalt/parser.py

@@ -37,7 +37,7 @@ def api_scan_configuration_hint(self):
             "be populated with the asset name while saving the configuration."
         )
 
-    def get_findings(self, file, test):
+    def get_findings(self, file, test, parser_custom_setting=None):
         if file is None:
             data = CobaltApiImporter().get_findings(test)
         else:

dojo/tools/api_edgescan/importer.py

@@ -8,7 +8,7 @@ class EdgescanImporter(object):
     Import from Edgescan API
     """
 
-    def get_findings(self, test):
+    def get_findings(self, test, parser_custom_setting=None):
         client, config = self.prepare_client(test)
         findings = client.get_findings(config.service_key_1)
         return findings

dojo/tools/api_edgescan/parser.py

@@ -31,7 +31,7 @@ def requires_tool_type(self, scan_type):
     def api_scan_configuration_hint(self):
         return "In the field <b>Service key 1</b>, provide the Edgescan asset ID(s). Leaving it blank will import all assets' findings."
 
-    def get_findings(self, file, test):
+    def get_findings(self, file, test, parser_custom_setting=None):
         if file:
             data = json.load(file)
         else:

dojo/tools/api_sonarqube/importer.py

@@ -20,7 +20,7 @@ class SonarQubeApiImporter(object):
      findings.
     """
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         items = self.import_issues(test)
         if settings.SONARQUBE_API_PARSER_HOTSPOTS:
             if items:

dojo/tools/api_sonarqube/parser.py

@@ -29,5 +29,5 @@ def api_scan_configuration_hint(self):
             "can be used for the Organization ID if using SonarCloud."
         )
 
-    def get_findings(self, json_output, test):
+    def get_findings(self, json_output, test, parser_custom_setting=None):
         return SonarQubeApiImporter().get_findings(json_output, test)

dojo/tools/api_vulners/importer.py

@@ -11,7 +11,7 @@ class VulnersImporter(object):
     Import from Vulners API
     """
 
-    def get_findings(self, test):
+    def get_findings(self, test, parser_custom_setting=None):
         client, config = self.prepare_client(test)
         findings = client.get_findings()
         return findings

dojo/tools/api_vulners/parser.py

@@ -38,7 +38,7 @@ def api_scan_configuration_hint(self):
     def requires_file(self, scan_type):
         return False
 
-    def get_findings(self, file, test):
+    def get_findings(self, file, test, parser_custom_setting=None):
         findings = []
 
         if file:

dojo/tools/appspider/parser.py

@@ -16,7 +16,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "AppSpider (Rapid7) - Use the VulnerabilitiesSummary.xml file found in the zipped report download."
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         if filename is None:
             return
 

dojo/tools/aqua/parser.py

@@ -13,7 +13,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return ""
 
-    def get_findings(self, json_output, test):
+    def get_findings(self, json_output, test, parser_custom_setting=None):
         tree = json.load(json_output)
         return self.get_items(tree, test)
 

dojo/tools/arachni/parser.py

@@ -23,7 +23,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Arachni JSON report format (generated with `arachni_reporter --reporter 'json'`)."
 
-    def get_findings(self, json_output, test):
+    def get_findings(self, json_output, test, parser_custom_setting=None):
         tree = json.load(json_output)
         return self.get_items(tree, test)
 

dojo/tools/asff/parser.py

@@ -26,7 +26,7 @@ def get_description_for_scan_types(self, scan_type):
         return """AWS Security Finding Format (ASFF).
         https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format-syntax.html"""
 
-    def get_findings(self, file, test):
+    def get_findings(self, file, test, parser_custom_setting=None):
         data = json.load(file)
         result = list()
         for item in data:

dojo/tools/auditjs/parser.py

@@ -32,7 +32,7 @@ def get_severity(self, cvss):
         else:
             return "Informational"
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         try:
             data = json.load(filename)
         except JSONDecodeError:

dojo/tools/aws_prowler/parser.py

@@ -20,7 +20,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Export of AWS Prowler in CSV or JSON format."
 
-    def get_findings(self, file, test):
+    def get_findings(self, file, test, parser_custom_setting=None):
         if file.name.lower().endswith(".csv"):
             return self.process_csv(file, test)
         elif file.name.lower().endswith(".json"):

dojo/tools/aws_prowler_v3/parser.py

@@ -19,7 +19,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Export of AWS Prowler JSON V3 format."
 
-    def get_findings(self, file, test):
+    def get_findings(self, file, test, parser_custom_setting=None):
         if file.name.lower().endswith('.json'):
             return self.process_json(file, test)
         else:

dojo/tools/aws_scout2/parser.py

@@ -20,7 +20,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "JS file in scout2-report/inc-awsconfig/aws_config.js."
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         content = filename.read()
         if isinstance(content, bytes):
             content = content.decode("utf-8")

dojo/tools/awssecurityhub/parser.py

@@ -14,7 +14,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "AWS Security Hub exports in JSON format."
 
-    def get_findings(self, filehandle, test):
+    def get_findings(self, filehandle, test, parser_custom_setting=None):
         tree = json.load(filehandle)
         if not isinstance(tree, dict):
             raise ValueError("Incorrect Security Hub report format")

dojo/tools/azure_security_center_recommendations/parser.py

@@ -18,7 +18,7 @@ def get_description_for_scan_types(self, scan_type):
             "CSV format."
         )
 
-    def get_findings(self, file, test):
+    def get_findings(self, file, test, parser_custom_setting=None):
         if file.name.lower().endswith(".csv"):
             return self.process_csv(file, test)
         else:

dojo/tools/bandit/parser.py

@@ -14,7 +14,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "JSON report format"
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         data = json.load(filename)
 
         results = list()

dojo/tools/blackduck/parser.py

@@ -20,7 +20,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Upload the zip file containing the security.csv and components.csv for Security and License risks."
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         normalized_findings = self.normalize_findings(filename)
         return self.ingest_findings(normalized_findings, test)
 

dojo/tools/blackduck_binary_analysis/parser.py

@@ -20,7 +20,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Blackduck Binary Analysis CSV file containing vulnerable binaries."
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         sorted_findings = self.sort_findings(filename)
         return self.ingest_findings(sorted_findings, test)
 

dojo/tools/blackduck_component_risk/parser.py

@@ -18,7 +18,7 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Upload the zip file containing the security.csv and files.csv."
 
-    def get_findings(self, filename, test):
+    def get_findings(self, filename, test, parser_custom_setting=None):
         """
         Function initializes the parser with a file and returns the items.
         :param filename: Input in Defect Dojo