From e7db24c860e5c014eeafd364ceb1d43a0b4a3d62 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Mon, 13 Nov 2023 22:32:54 +0100 Subject: [PATCH 1/8] :tada: added humble --- .../en/integrations/parsers/file/humble.md | 6 ++ dojo/tools/humble/__init__.py | 0 dojo/tools/humble/parser.py | 35 ++++++++++++ unittests/scans/humble/many_findings.json | 55 +++++++++++++++++++ unittests/scans/humble/no_findings.json | 38 +++++++++++++ unittests/tools/test_humble_parser.py | 11 ++++ 6 files changed, 145 insertions(+) create mode 100644 docs/content/en/integrations/parsers/file/humble.md create mode 100644 dojo/tools/humble/__init__.py create mode 100644 dojo/tools/humble/parser.py create mode 100644 unittests/scans/humble/many_findings.json create mode 100644 unittests/scans/humble/no_findings.json create mode 100644 unittests/tools/test_humble_parser.py diff --git a/docs/content/en/integrations/parsers/file/humble.md b/docs/content/en/integrations/parsers/file/humble.md new file mode 100644 index 00000000000..85842c57ab4 --- /dev/null +++ b/docs/content/en/integrations/parsers/file/humble.md @@ -0,0 +1,6 @@ +--- +title: "Humble Report" +toc_hide: true +--- +Import JSON report of the Humble scanner + \ No newline at end of file diff --git a/dojo/tools/humble/__init__.py b/dojo/tools/humble/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dojo/tools/humble/parser.py b/dojo/tools/humble/parser.py new file mode 100644 index 00000000000..2eeb740e0f1 --- /dev/null +++ b/dojo/tools/humble/parser.py @@ -0,0 +1,35 @@ +import json +from dojo.models import Finding, Endpoint + + +class HumbleParser(object): + """Humble (https://github.com/rfc-st/humble)""" + + def get_scan_types(self): + return ["Humble Json Importer"] + + def get_label_for_scan_types(self, scan_type): + return scan_type # no custom label for now + + def get_description_for_scan_types(self, scan_type): + return "JSON output of Humble scan." + + def return_finding(self, test, finding, url=None): + print("TODO") + + def get_findings(self, filename, test): + items = [] + try: + data = json.load(filename) + except ValueError as err: + data = {} + if data != {} and data[0].get("url") is not None: + for item in data: + url = item["url"] + for finding in item["report"]: + items.append(self.return_finding(test=test, finding=finding, url=url)) + return items + else: + for finding in data: + items.append(self.return_finding(test=test, finding=finding)) + return items diff --git a/unittests/scans/humble/many_findings.json b/unittests/scans/humble/many_findings.json new file mode 100644 index 00000000000..a1965d42d3a --- /dev/null +++ b/unittests/scans/humble/many_findings.json @@ -0,0 +1,55 @@ +{ + "[0. Info]": { + "Date": "2023/11/13 - 09:20:17", + "URL": "https://asdf.asf.hs" + }, + "[HTTP Response Headers]": { + "Cache-Control": "no-store, no-cache, must-revalidate, post-check=0, pre-check=0", + "Connection": "Keep-Alive", + "Content-Security-Policy": "script-src 'self';", + "Content-Type": "text/html; charset=utf-8", + "Date": "Mon, 13 Nov 2023 08:20:19 GMT", + "Expires": "Wed, 17 Aug 2005 00:00:00 GMT", + "Keep-Alive": "timeout=5, max=100", + "Last-Modified": "Mon, 13 Nov 2023 08:20:19 GMT", + "Permissions-Policy": "interest-cohort=()", + "Pragma": "no-cache", + "Referrer-Policy": "strict-origin", + "Set-Cookie": "8caa9ddc6c754d479548a7ce5ac1adbd=t2l01hama41hfcpn38puuhfbjo; path=/; HttpOnly, BIGipServerPool_gqmc=919077386.47873.0000; path=/; Httponly; Secure, TS01cefe6e=01eecca7b203023b1263e1cdfac41fe36c7a6cc4f07b7373afac56e5205f2af6e164c6ed8f9523e967dc229dce1122fd9d5f287c6d08455ec6aa42d9987e2e8ce5d3bb4656e121044701094be65114d1dd5e46997e; Path=/;", + "Strict-Transport-Security": "max-age=31536000; includeSubDomain$", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "sameorigin", + "X-XSS-Protection": "1; mode=block" + }, + "[1. Missing HTTP Security Headers]": [ + "Clear-Site-Data", + "Cross-Origin-Embedder-Policy", + "Cross-Origin-Opener-Policy", + "Cross-Origin-Resource-Policy", + "NEL", + "X-Permitted-Cross-Domain-Policies" + ], + "[2. Fingerprint HTTP Response Headers]": [ + "Nothing to report, all seems OK!" + ], + "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]": [ + "Pragma (Deprecated Header)", + "Strict-Transport-Security (Recommended Values)", + "X-XSS-Protection (Unsafe Value)" + ], + "[4. Empty HTTP Response Headers Values]": [ + "Nothing to report, all seems OK!" + ], + "[5. Browser Compatibility for Enabled HTTP Security Headers]": { + "Cache-Control": "https://caniuse.com/?search=Cache-Control", + "Content-Type": "https://caniuse.com/?search=Content-Type", + "Content-Security-Policy": "https://caniuse.com/?search=contentsecuritypolicy2", + "Permissions-Policy": "https://caniuse.com/?search=Permissions-Policy", + "Referrer-Policy": "https://caniuse.com/?search=Referrer-Policy", + "Strict-Transport-Security": "https://caniuse.com/?search=Strict-Transport-Security", + "X-Content-Type-Options": "https://caniuse.com/?search=X-Content-Type-Options", + "X-Frame-Options": "https://caniuse.com/?search=X-Frame-Options" + } +} \ No newline at end of file diff --git a/unittests/scans/humble/no_findings.json b/unittests/scans/humble/no_findings.json new file mode 100644 index 00000000000..bd824c9068c --- /dev/null +++ b/unittests/scans/humble/no_findings.json @@ -0,0 +1,38 @@ +{ + "[0. Info]": { + "Date": "2023/11/13 - 22:26:04", + "URL": "https://asfasdfasdfgqmc.gov.ph" + }, + "[HTTP Response Headers]": { + "": "" + }, + "[1. Missing HTTP Security Headers]": [ + "Cache-Control", + "Clear-Site-Data", + "Content-Type", + "Cross-Origin-Embedder-Policy", + "Cross-Origin-Opener-Policy", + "Cross-Origin-Resource-Policy", + "Content-Security-Policy", + "NEL", + "Permissions-Policy", + "Referrer-Policy", + "Strict-Transport-Security", + "X-Content-Type-Options", + "X-Permitted-Cross-Domain-Policies", + "X-Frame-Options", + "X-Frame-Options" + ], + "[2. Fingerprint HTTP Response Headers]": [ + "Nothing to report, all seems OK!" + ], + "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]": [ + "Nothing to report, all seems OK!" + ], + "[4. Empty HTTP Response Headers Values]": [ + "Nothing to report, all seems OK!" + ], + "[5. Browser Compatibility for Enabled HTTP Security Headers]": { + "No HTTP security headers are enabled.": "" + } +}% \ No newline at end of file diff --git a/unittests/tools/test_humble_parser.py b/unittests/tools/test_humble_parser.py new file mode 100644 index 00000000000..7e1b524e3cd --- /dev/null +++ b/unittests/tools/test_humble_parser.py @@ -0,0 +1,11 @@ +from dojo.tools.humble.parser import HumbleParser +from dojo.models import Test, Finding +from unittests.dojo_test_case import DojoTestCase + + +class TestHumbleParser(DojoTestCase): + def test_hydra_parser_with_many_findings_has_many_findings(self): + testfile = open("unittests/scans/humble/many_findings.json") + parser = HumbleParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() \ No newline at end of file From d6f57d3e2c5507386192847bd29526f3209fd1a6 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Tue, 14 Nov 2023 23:17:36 +0100 Subject: [PATCH 2/8] fixed humble --- .../en/integrations/parsers/file/humble.md | 2 +- dojo/tools/humble/parser.py | 54 ++++++++++++++----- unittests/scans/humble/no_findings.json | 38 ------------- unittests/tools/test_humble_parser.py | 11 ++-- 4 files changed, 49 insertions(+), 56 deletions(-) delete mode 100644 unittests/scans/humble/no_findings.json diff --git a/docs/content/en/integrations/parsers/file/humble.md b/docs/content/en/integrations/parsers/file/humble.md index 85842c57ab4..56c3f73b52e 100644 --- a/docs/content/en/integrations/parsers/file/humble.md +++ b/docs/content/en/integrations/parsers/file/humble.md @@ -3,4 +3,4 @@ title: "Humble Report" toc_hide: true --- Import JSON report of the Humble scanner - \ No newline at end of file + \ No newline at end of file diff --git a/dojo/tools/humble/parser.py b/dojo/tools/humble/parser.py index 2eeb740e0f1..03cd8c82fef 100644 --- a/dojo/tools/humble/parser.py +++ b/dojo/tools/humble/parser.py @@ -1,5 +1,5 @@ import json -from dojo.models import Finding, Endpoint +from dojo.models import Finding class HumbleParser(object): @@ -14,22 +14,48 @@ def get_label_for_scan_types(self, scan_type): def get_description_for_scan_types(self, scan_type): return "JSON output of Humble scan." - def return_finding(self, test, finding, url=None): - print("TODO") - def get_findings(self, filename, test): items = [] try: data = json.load(filename) except ValueError as err: data = {} - if data != {} and data[0].get("url") is not None: - for item in data: - url = item["url"] - for finding in item["report"]: - items.append(self.return_finding(test=test, finding=finding, url=url)) - return items - else: - for finding in data: - items.append(self.return_finding(test=test, finding=finding)) - return items + if data != {}: + url = data['[0. Info]']['URL'] + for content in data['[1. Missing HTTP Security Headers]']: + if content != "Nothing to report, all seems OK!": + finding = Finding(title=url + "_missing_" + str(content), + test=test, + description="This security Header is missing: " + content, + severity="Medium", + static_finding=False, + dynamic_finding=True) + items.append(finding) + for content in data['[2. Fingerprint HTTP Response Headers]']: + if content != "Nothing to report, all seems OK!": + finding = Finding(title=url + "_fingerprint_" + str(content), + test=test, + description="This fingerprint HTTP Response Header is available. Please remove it: " + content, + severity="Medium", + static_finding=False, + dynamic_finding=True) + items.append(finding) + for content in data['[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]']: + if content != "Nothing to report, all seems OK!": + finding = Finding(title=url + "_deprecatedheader_" + str(content), + test=test, + description="This deprecated HTTP Response Header is available. Please remove it: " + content, + severity="Medium", + static_finding=False, + dynamic_finding=True) + items.append(finding) + for content in data['[4. Empty HTTP Response Headers Values]']: + if content != "Nothing to report, all seems OK!": + finding = Finding(title=url + "_emptyhttpresponse_" + str(content), + test=test, + description="This empty HTTP Response Header value is available. Please remove it: " + content, + severity="Medium", + static_finding=False, + dynamic_finding=True) + items.append(finding) + return items diff --git a/unittests/scans/humble/no_findings.json b/unittests/scans/humble/no_findings.json deleted file mode 100644 index bd824c9068c..00000000000 --- a/unittests/scans/humble/no_findings.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "[0. Info]": { - "Date": "2023/11/13 - 22:26:04", - "URL": "https://asfasdfasdfgqmc.gov.ph" - }, - "[HTTP Response Headers]": { - "": "" - }, - "[1. Missing HTTP Security Headers]": [ - "Cache-Control", - "Clear-Site-Data", - "Content-Type", - "Cross-Origin-Embedder-Policy", - "Cross-Origin-Opener-Policy", - "Cross-Origin-Resource-Policy", - "Content-Security-Policy", - "NEL", - "Permissions-Policy", - "Referrer-Policy", - "Strict-Transport-Security", - "X-Content-Type-Options", - "X-Permitted-Cross-Domain-Policies", - "X-Frame-Options", - "X-Frame-Options" - ], - "[2. Fingerprint HTTP Response Headers]": [ - "Nothing to report, all seems OK!" - ], - "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]": [ - "Nothing to report, all seems OK!" - ], - "[4. Empty HTTP Response Headers Values]": [ - "Nothing to report, all seems OK!" - ], - "[5. Browser Compatibility for Enabled HTTP Security Headers]": { - "No HTTP security headers are enabled.": "" - } -}% \ No newline at end of file diff --git a/unittests/tools/test_humble_parser.py b/unittests/tools/test_humble_parser.py index 7e1b524e3cd..30af537d75c 100644 --- a/unittests/tools/test_humble_parser.py +++ b/unittests/tools/test_humble_parser.py @@ -1,11 +1,16 @@ from dojo.tools.humble.parser import HumbleParser -from dojo.models import Test, Finding +from dojo.models import Test from unittests.dojo_test_case import DojoTestCase class TestHumbleParser(DojoTestCase): - def test_hydra_parser_with_many_findings_has_many_findings(self): + def test_humble_parser_with_many_findings(self): testfile = open("unittests/scans/humble/many_findings.json") parser = HumbleParser() findings = parser.get_findings(testfile, Test()) - testfile.close() \ No newline at end of file + testfile.close() + self.assertEqual(9, len(findings)) + finding = findings[0] + self.assertEqual("https://asdf.asf.hs_missing_Clear-Site-Data", finding.title) + finding = findings[7] + self.assertEqual("https://asdf.asf.hs_deprecatedheader_Strict-Transport-Security (Recommended Values)", finding.title) From b7184e4b48011942e6f08dfe11e3274f39d2bf85 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Wed, 15 Nov 2023 08:55:36 +0100 Subject: [PATCH 3/8] added endpoints --- dojo/tools/humble/parser.py | 14 ++++++- unittests/scans/humble/many_findings.json | 1 - unittests/scans/humble/many_findings2.json | 49 ++++++++++++++++++++++ unittests/tools/test_humble_parser.py | 13 ++++++ 4 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 unittests/scans/humble/many_findings2.json diff --git a/dojo/tools/humble/parser.py b/dojo/tools/humble/parser.py index 03cd8c82fef..20975acd68d 100644 --- a/dojo/tools/humble/parser.py +++ b/dojo/tools/humble/parser.py @@ -1,5 +1,5 @@ import json -from dojo.models import Finding +from dojo.models import Finding, Endpoint class HumbleParser(object): @@ -31,6 +31,9 @@ def get_findings(self, filename, test): static_finding=False, dynamic_finding=True) items.append(finding) + finding.unsaved_endpoints = list() + endpoint = Endpoint(host=url) + finding.unsaved_endpoints.append(endpoint) for content in data['[2. Fingerprint HTTP Response Headers]']: if content != "Nothing to report, all seems OK!": finding = Finding(title=url + "_fingerprint_" + str(content), @@ -40,6 +43,9 @@ def get_findings(self, filename, test): static_finding=False, dynamic_finding=True) items.append(finding) + finding.unsaved_endpoints = list() + endpoint = Endpoint(host=url) + finding.unsaved_endpoints.append(endpoint) for content in data['[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]']: if content != "Nothing to report, all seems OK!": finding = Finding(title=url + "_deprecatedheader_" + str(content), @@ -49,6 +55,9 @@ def get_findings(self, filename, test): static_finding=False, dynamic_finding=True) items.append(finding) + finding.unsaved_endpoints = list() + endpoint = Endpoint(host=url) + finding.unsaved_endpoints.append(endpoint) for content in data['[4. Empty HTTP Response Headers Values]']: if content != "Nothing to report, all seems OK!": finding = Finding(title=url + "_emptyhttpresponse_" + str(content), @@ -58,4 +67,7 @@ def get_findings(self, filename, test): static_finding=False, dynamic_finding=True) items.append(finding) + finding.unsaved_endpoints = list() + endpoint = Endpoint(host=url) + finding.unsaved_endpoints.append(endpoint) return items diff --git a/unittests/scans/humble/many_findings.json b/unittests/scans/humble/many_findings.json index a1965d42d3a..82a81611939 100644 --- a/unittests/scans/humble/many_findings.json +++ b/unittests/scans/humble/many_findings.json @@ -15,7 +15,6 @@ "Permissions-Policy": "interest-cohort=()", "Pragma": "no-cache", "Referrer-Policy": "strict-origin", - "Set-Cookie": "8caa9ddc6c754d479548a7ce5ac1adbd=t2l01hama41hfcpn38puuhfbjo; path=/; HttpOnly, BIGipServerPool_gqmc=919077386.47873.0000; path=/; Httponly; Secure, TS01cefe6e=01eecca7b203023b1263e1cdfac41fe36c7a6cc4f07b7373afac56e5205f2af6e164c6ed8f9523e967dc229dce1122fd9d5f287c6d08455ec6aa42d9987e2e8ce5d3bb4656e121044701094be65114d1dd5e46997e; Path=/;", "Strict-Transport-Security": "max-age=31536000; includeSubDomain$", "Transfer-Encoding": "chunked", "Vary": "Accept-Encoding", diff --git a/unittests/scans/humble/many_findings2.json b/unittests/scans/humble/many_findings2.json new file mode 100644 index 00000000000..b532bb6c344 --- /dev/null +++ b/unittests/scans/humble/many_findings2.json @@ -0,0 +1,49 @@ +{ + "[0. Info]": { + "Date": "2023/11/15 - 08:42:38", + "URL": "https://testestset.com" + }, + "[HTTP Response Headers]": { + "CF-Cache-Status": "DYNAMIC", + "CF-RAY": "8265dbd49d362bde-FRA", + "Cache-Control": "no-store, private", + "Connection": "keep-alive", + "Content-Encoding": "gzip", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Wed, 15 Nov 2023 07:42:39 GMT", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "X-Content-Type-Options": "nosniff", + "X-UA-Compatible": "IE=edge" + }, + "[1. Missing HTTP Security Headers]": [ + "Clear-Site-Data", + "Cross-Origin-Embedder-Policy", + "Cross-Origin-Opener-Policy", + "Cross-Origin-Resource-Policy", + "Content-Security-Policy", + "NEL", + "Permissions-Policy", + "Referrer-Policy", + "Strict-Transport-Security", + "X-Permitted-Cross-Domain-Policies", + "X-Frame-Options" + ], + "[2. Fingerprint HTTP Response Headers]": [ + "Cf-Cache-Status", + "Cf-Ray", + "Server" + ], + "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]": [ + "Cache-Control (Recommended Values)", + "X-UA-compatible (Deprecated Header)" + ], + "[4. Empty HTTP Response Headers Values]": [ + "Nothing to report, all seems OK!" + ], + "[5. Browser Compatibility for Enabled HTTP Security Headers]": { + "Cache-Control": "https://caniuse.com/?search=Cache-Control", + "Content-Type": "https://caniuse.com/?search=Content-Type", + "X-Content-Type-Options": "https://caniuse.com/?search=X-Content-Type-Options" + } +} \ No newline at end of file diff --git a/unittests/tools/test_humble_parser.py b/unittests/tools/test_humble_parser.py index 30af537d75c..632304c75c1 100644 --- a/unittests/tools/test_humble_parser.py +++ b/unittests/tools/test_humble_parser.py @@ -11,6 +11,19 @@ def test_humble_parser_with_many_findings(self): testfile.close() self.assertEqual(9, len(findings)) finding = findings[0] + self.assertEqual(finding.unsaved_endpoints[0].host, "https://asdf.asf.hs") self.assertEqual("https://asdf.asf.hs_missing_Clear-Site-Data", finding.title) finding = findings[7] self.assertEqual("https://asdf.asf.hs_deprecatedheader_Strict-Transport-Security (Recommended Values)", finding.title) + + def test_humble_parser_with_many_findings2(self): + testfile = open("unittests/scans/humble/many_findings2.json") + parser = HumbleParser() + findings = parser.get_findings(testfile, Test()) + testfile.close() + self.assertEqual(16, len(findings)) + finding = findings[0] + self.assertEqual("https://testestset.com_missing_Clear-Site-Data", finding.title) + finding = findings[7] + self.assertEqual("https://testestset.com_missing_Referrer-Policy", finding.title) + self.assertEqual("This security Header is missing: Referrer-Policy", finding.description) From 15947586418209a6c29a315e0cd7a82f074b9227 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Wed, 15 Nov 2023 11:49:42 +0100 Subject: [PATCH 4/8] fix according to comment --- dojo/tools/humble/parser.py | 19 +++++++++++-------- unittests/scans/humble/many_findings2.json | 2 +- unittests/tools/test_humble_parser.py | 17 ++++++++++++----- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/dojo/tools/humble/parser.py b/dojo/tools/humble/parser.py index 20975acd68d..63f94914c44 100644 --- a/dojo/tools/humble/parser.py +++ b/dojo/tools/humble/parser.py @@ -22,9 +22,12 @@ def get_findings(self, filename, test): data = {} if data != {}: url = data['[0. Info]']['URL'] + import re + host = re.compile(r"https?://(www\.)?") + host = host.sub('', url).strip().strip('/') for content in data['[1. Missing HTTP Security Headers]']: if content != "Nothing to report, all seems OK!": - finding = Finding(title=url + "_missing_" + str(content), + finding = Finding(title=host + "_missing_" + str(content), test=test, description="This security Header is missing: " + content, severity="Medium", @@ -32,11 +35,11 @@ def get_findings(self, filename, test): dynamic_finding=True) items.append(finding) finding.unsaved_endpoints = list() - endpoint = Endpoint(host=url) + endpoint = Endpoint(host=host) finding.unsaved_endpoints.append(endpoint) for content in data['[2. Fingerprint HTTP Response Headers]']: if content != "Nothing to report, all seems OK!": - finding = Finding(title=url + "_fingerprint_" + str(content), + finding = Finding(title=host + "_fingerprint_" + str(content), test=test, description="This fingerprint HTTP Response Header is available. Please remove it: " + content, severity="Medium", @@ -44,11 +47,11 @@ def get_findings(self, filename, test): dynamic_finding=True) items.append(finding) finding.unsaved_endpoints = list() - endpoint = Endpoint(host=url) + endpoint = Endpoint(host=host) finding.unsaved_endpoints.append(endpoint) for content in data['[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]']: if content != "Nothing to report, all seems OK!": - finding = Finding(title=url + "_deprecatedheader_" + str(content), + finding = Finding(title=host + "_deprecatedheader_" + str(content), test=test, description="This deprecated HTTP Response Header is available. Please remove it: " + content, severity="Medium", @@ -56,11 +59,11 @@ def get_findings(self, filename, test): dynamic_finding=True) items.append(finding) finding.unsaved_endpoints = list() - endpoint = Endpoint(host=url) + endpoint = Endpoint(host=host) finding.unsaved_endpoints.append(endpoint) for content in data['[4. Empty HTTP Response Headers Values]']: if content != "Nothing to report, all seems OK!": - finding = Finding(title=url + "_emptyhttpresponse_" + str(content), + finding = Finding(title=host + "_emptyhttpresponse_" + str(content), test=test, description="This empty HTTP Response Header value is available. Please remove it: " + content, severity="Medium", @@ -68,6 +71,6 @@ def get_findings(self, filename, test): dynamic_finding=True) items.append(finding) finding.unsaved_endpoints = list() - endpoint = Endpoint(host=url) + endpoint = Endpoint(host=host) finding.unsaved_endpoints.append(endpoint) return items diff --git a/unittests/scans/humble/many_findings2.json b/unittests/scans/humble/many_findings2.json index b532bb6c344..68f60db55a9 100644 --- a/unittests/scans/humble/many_findings2.json +++ b/unittests/scans/humble/many_findings2.json @@ -1,7 +1,7 @@ { "[0. Info]": { "Date": "2023/11/15 - 08:42:38", - "URL": "https://testestset.com" + "URL": "http://testestset.com" }, "[HTTP Response Headers]": { "CF-Cache-Status": "DYNAMIC", diff --git a/unittests/tools/test_humble_parser.py b/unittests/tools/test_humble_parser.py index 632304c75c1..49a6fb071cb 100644 --- a/unittests/tools/test_humble_parser.py +++ b/unittests/tools/test_humble_parser.py @@ -8,22 +8,29 @@ def test_humble_parser_with_many_findings(self): testfile = open("unittests/scans/humble/many_findings.json") parser = HumbleParser() findings = parser.get_findings(testfile, Test()) + for finding in findings: + for endpoint in finding.unsaved_endpoints: + endpoint.clean() testfile.close() self.assertEqual(9, len(findings)) finding = findings[0] - self.assertEqual(finding.unsaved_endpoints[0].host, "https://asdf.asf.hs") - self.assertEqual("https://asdf.asf.hs_missing_Clear-Site-Data", finding.title) + self.assertEqual(finding.unsaved_endpoints[0].host, "asdf.asf.hs") + self.assertEqual("asdf.asf.hs_missing_Clear-Site-Data", finding.title) finding = findings[7] - self.assertEqual("https://asdf.asf.hs_deprecatedheader_Strict-Transport-Security (Recommended Values)", finding.title) + self.assertEqual("asdf.asf.hs_deprecatedheader_Strict-Transport-Security (Recommended Values)", finding.title) def test_humble_parser_with_many_findings2(self): testfile = open("unittests/scans/humble/many_findings2.json") parser = HumbleParser() findings = parser.get_findings(testfile, Test()) + for finding in findings: + for endpoint in finding.unsaved_endpoints: + endpoint.clean() testfile.close() self.assertEqual(16, len(findings)) finding = findings[0] - self.assertEqual("https://testestset.com_missing_Clear-Site-Data", finding.title) + self.assertEqual(finding.unsaved_endpoints[0].host, "testestset.com") + self.assertEqual("testestset.com_missing_Clear-Site-Data", finding.title) finding = findings[7] - self.assertEqual("https://testestset.com_missing_Referrer-Policy", finding.title) + self.assertEqual("testestset.com_missing_Referrer-Policy", finding.title) self.assertEqual("This security Header is missing: Referrer-Policy", finding.description) From 9da2efc9b362248cefeaa702bc3a4a041fa0b7f7 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Fri, 17 Nov 2023 23:58:21 +0100 Subject: [PATCH 5/8] fix according to review --- dojo/tools/humble/parser.py | 31 +++++++-------------------- unittests/tools/test_humble_parser.py | 8 +++---- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/dojo/tools/humble/parser.py b/dojo/tools/humble/parser.py index 63f94914c44..689ce080187 100644 --- a/dojo/tools/humble/parser.py +++ b/dojo/tools/humble/parser.py @@ -22,55 +22,40 @@ def get_findings(self, filename, test): data = {} if data != {}: url = data['[0. Info]']['URL'] - import re - host = re.compile(r"https?://(www\.)?") - host = host.sub('', url).strip().strip('/') for content in data['[1. Missing HTTP Security Headers]']: if content != "Nothing to report, all seems OK!": - finding = Finding(title=host + "_missing_" + str(content), - test=test, + finding = Finding(title="Missing header: " + str(content), description="This security Header is missing: " + content, severity="Medium", static_finding=False, dynamic_finding=True) items.append(finding) - finding.unsaved_endpoints = list() - endpoint = Endpoint(host=host) - finding.unsaved_endpoints.append(endpoint) + finding.unsaved_endpoints = [Endpoint.from_uri(url)] for content in data['[2. Fingerprint HTTP Response Headers]']: if content != "Nothing to report, all seems OK!": - finding = Finding(title=host + "_fingerprint_" + str(content), - test=test, + finding = Finding(title="Available fingerprint:" + str(content), description="This fingerprint HTTP Response Header is available. Please remove it: " + content, severity="Medium", static_finding=False, dynamic_finding=True) items.append(finding) - finding.unsaved_endpoints = list() - endpoint = Endpoint(host=host) - finding.unsaved_endpoints.append(endpoint) + finding.unsaved_endpoints = [Endpoint.from_uri(url)] for content in data['[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]']: if content != "Nothing to report, all seems OK!": - finding = Finding(title=host + "_deprecatedheader_" + str(content), - test=test, + finding = Finding(title="Deprecated header: " + str(content), description="This deprecated HTTP Response Header is available. Please remove it: " + content, severity="Medium", static_finding=False, dynamic_finding=True) items.append(finding) - finding.unsaved_endpoints = list() - endpoint = Endpoint(host=host) - finding.unsaved_endpoints.append(endpoint) + finding.unsaved_endpoints = [Endpoint.from_uri(url)] for content in data['[4. Empty HTTP Response Headers Values]']: if content != "Nothing to report, all seems OK!": - finding = Finding(title=host + "_emptyhttpresponse_" + str(content), - test=test, + finding = Finding(title="Empty HTTP response header: " + str(content), description="This empty HTTP Response Header value is available. Please remove it: " + content, severity="Medium", static_finding=False, dynamic_finding=True) items.append(finding) - finding.unsaved_endpoints = list() - endpoint = Endpoint(host=host) - finding.unsaved_endpoints.append(endpoint) + finding.unsaved_endpoints = [Endpoint.from_uri(url)] return items diff --git a/unittests/tools/test_humble_parser.py b/unittests/tools/test_humble_parser.py index 49a6fb071cb..ccd99d44373 100644 --- a/unittests/tools/test_humble_parser.py +++ b/unittests/tools/test_humble_parser.py @@ -15,9 +15,9 @@ def test_humble_parser_with_many_findings(self): self.assertEqual(9, len(findings)) finding = findings[0] self.assertEqual(finding.unsaved_endpoints[0].host, "asdf.asf.hs") - self.assertEqual("asdf.asf.hs_missing_Clear-Site-Data", finding.title) + self.assertEqual("Missing header: Clear-Site-Data", finding.title) finding = findings[7] - self.assertEqual("asdf.asf.hs_deprecatedheader_Strict-Transport-Security (Recommended Values)", finding.title) + self.assertEqual("Deprecated header: Strict-Transport-Security (Recommended Values)", finding.title) def test_humble_parser_with_many_findings2(self): testfile = open("unittests/scans/humble/many_findings2.json") @@ -30,7 +30,7 @@ def test_humble_parser_with_many_findings2(self): self.assertEqual(16, len(findings)) finding = findings[0] self.assertEqual(finding.unsaved_endpoints[0].host, "testestset.com") - self.assertEqual("testestset.com_missing_Clear-Site-Data", finding.title) + self.assertEqual("Missing header: Clear-Site-Data", finding.title) finding = findings[7] - self.assertEqual("testestset.com_missing_Referrer-Policy", finding.title) + self.assertEqual("Missing header: Referrer-Policy", finding.title) self.assertEqual("This security Header is missing: Referrer-Policy", finding.description) From 4bf9e061b59e93d56a373ab95fc7a32a96bc3849 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Sun, 19 Nov 2023 12:43:55 +0100 Subject: [PATCH 6/8] update --- dojo/tools/humble/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dojo/tools/humble/__init__.py b/dojo/tools/humble/__init__.py index e69de29bb2d..99e8e118c6a 100644 --- a/dojo/tools/humble/__init__.py +++ b/dojo/tools/humble/__init__.py @@ -0,0 +1 @@ +__author__ = "manuel_sommer" From 502346b148471d78d585aa50d211f7abc4baa676 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Tue, 28 Nov 2023 08:22:09 +0100 Subject: [PATCH 7/8] added deduplication setting --- dojo/settings/settings.dist.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 4f0716a5549..0407054ac2f 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -1260,6 +1260,7 @@ def saml2_attrib_map_format(dict): 'KubeHunter Scan': ['title', 'description'], 'kube-bench Scan': ['title', 'vuln_id_from_tool', 'description'], 'Threagile risks report': ['title', 'cwe', "severity"], + 'Humble Json Importer': ['title', 'description'], } # Override the hardcoded settings here via the env var @@ -1457,6 +1458,7 @@ def saml2_attrib_map_format(dict): 'KubeHunter Scan': DEDUPE_ALGO_HASH_CODE, 'kube-bench Scan': DEDUPE_ALGO_HASH_CODE, 'Threagile risks report': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL_OR_HASH_CODE, + 'Humble Json Importer': DEDUPE_ALGO_HASH_CODE, } # Override the hardcoded settings here via the env var From d029670453c1e03865bf439d0bf3dad1be922c14 Mon Sep 17 00:00:00 2001 From: Manuel Sommer Date: Tue, 28 Nov 2023 10:36:06 +0100 Subject: [PATCH 8/8] fix --- dojo/settings/settings.dist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 0407054ac2f..25740feb409 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -1260,7 +1260,7 @@ def saml2_attrib_map_format(dict): 'KubeHunter Scan': ['title', 'description'], 'kube-bench Scan': ['title', 'vuln_id_from_tool', 'description'], 'Threagile risks report': ['title', 'cwe', "severity"], - 'Humble Json Importer': ['title', 'description'], + 'Humble Json Importer': ['title'], } # Override the hardcoded settings here via the env var