From 3ee9f2d1fa2e3ac052060cea1178ffb02f25984b Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 3 Mar 2025 15:15:14 +0100 Subject: [PATCH 01/12] Severity field in IDF Severity is expected in IntelMQ for a long time and partially, it's already used by e.g. ShadowServer reports. This implementation is based on their understanding of the field, but with explicit mentioning that operators could adjust it based on their knowledge. This is not intended to be an ultimate severity classification, but a help for first triage of recived events. Close #2365 --- CHANGELOG.md | 3 ++- intelmq/etc/harmonization.conf | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a5ff90258..06de26e1b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,6 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ### Development ### Data Format - - Implementing [IEP009](https://github.com/certtools/ieps/tree/main/009) introducing fields to identify products and vulnerabilities: `product.full_name`, `product.name`, `product.vendor`, `product.version`, `product.vulnerabilities`. To store in existing PostgreSQL instances, a following @@ -40,6 +39,8 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ALTER TABLE events ADD "product.version" text; ALTER TABLE events ADD "product.vulnerabilities" text; ``` +- added `severity` field to help with triaging received events (PR# by Kamil Mańkowski). + To allow saving the field in PostgreSQL database in existing installations, the following schema update is necessary: `ALTER TABLE events ADD severity varchar(10);`. ### Bots #### Collectors diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index b9e57d6e22..069a39e6be 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -382,6 +382,12 @@ "tlp": { "description": "Traffic Light Protocol level of the event.", "type": "TLP" + }, + "severity": { + "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", + "type": "LowercaseString", + "regex": "^(critical|high|medium|low|info|undefined)$", + "length": 10 } }, "report": { From cdd4372b55ef627de7e2beafa2bb2e019f609aaf Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 3 Mar 2025 17:26:39 +0100 Subject: [PATCH 02/12] Fix order --- intelmq/etc/harmonization.conf | 12 ++++++------ intelmq/tests/bin/initdb.sql | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index 069a39e6be..e25a938720 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -253,6 +253,12 @@ "description": "Some source may report URLs related to a an image generated of a resource without any metadata. Or an URL pointing to resource, which has been rendered into a webshot, e.g. a PNG image and the relevant metadata related to its retrieval/generation.", "type": "URL" }, + "severity": { + "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", + "length": 10, + "regex": "^(critical|high|medium|low|info|undefined)$", + "type": "LowercaseString" + }, "source.abuse_contact": { "description": "Abuse contact for source address. A comma separated list.", "type": "LowercaseString" @@ -382,12 +388,6 @@ "tlp": { "description": "Traffic Light Protocol level of the event.", "type": "TLP" - }, - "severity": { - "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", - "type": "LowercaseString", - "regex": "^(critical|high|medium|low|info|undefined)$", - "length": 10 } }, "report": { diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index 19faecb855..cbf1a8d1c1 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -57,6 +57,7 @@ CREATE TABLE events ( "raw" text, "rtir_id" integer, "screenshot_url" text, + "severity" varchar(10), "source.abuse_contact" text, "source.account" text, "source.allocated" timestamp with time zone, @@ -98,4 +99,4 @@ CREATE INDEX "idx_events_source.asn" ON events USING btree ("source.asn"); CREATE INDEX "idx_events_source.ip" ON events USING btree ("source.ip"); CREATE INDEX "idx_events_source.fqdn" ON events USING btree ("source.fqdn"); CREATE INDEX "idx_events_time.observation" ON events USING btree ("time.observation"); -CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); +CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); \ No newline at end of file From d58986760160e884a656f640c5d5aa93a1ae837d Mon Sep 17 00:00:00 2001 From: kamil-certat <117654481+kamil-certat@users.noreply.github.com> Date: Thu, 3 Apr 2025 16:07:14 +0200 Subject: [PATCH 03/12] Update CHANGELOG.md Co-authored-by: Sebastian --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06de26e1b6..73058125ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,7 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ALTER TABLE events ADD "product.version" text; ALTER TABLE events ADD "product.vulnerabilities" text; ``` -- added `severity` field to help with triaging received events (PR# by Kamil Mańkowski). +- added `severity` field to help with triaging received events (PR#2575 by Kamil Mańkowski). To allow saving the field in PostgreSQL database in existing installations, the following schema update is necessary: `ALTER TABLE events ADD severity varchar(10);`. ### Bots From db2001e6cf48736a081b12ba51e8ecea60422ccf Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 3 Apr 2025 17:18:02 +0200 Subject: [PATCH 04/12] Added news entry and upgrade function --- NEWS.md | 1 + intelmq/lib/upgrades.py | 12 ++++++++++++ intelmq/tests/bin/initdb.sql | 2 +- intelmq/tests/lib/test_upgrades.py | 3 +++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 4c948919e9..27e3b20aca 100644 --- a/NEWS.md +++ b/NEWS.md @@ -27,6 +27,7 @@ ALTER TABLE events ADD "product.name" text; ALTER TABLE events ADD "product.vendor" text; ALTER TABLE events ADD "product.version" text; ALTER TABLE events ADD "product.vulnerabilities" text; +ALTER TABLE events ADD severity varchar(10); ``` ### Configuration diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 3da3e95fc9..ba8853401c 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -975,6 +975,18 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): message = f"Found discontinued Twitter collector bot: {', '.join(found_twitter_collector)}" return message or changed, configuration, harmonization +def v341_new_fields(configuration, harmonization, dry_run, **kwargs): + changed = False + if "severity" not in harmonization["event"]: + harmonization["event"]["severity"] = { + "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", + "length": 10, + "regex": "^(critical|high|medium|low|info|undefined)$", + "type": "LowercaseString", + } + changed = True + return changed, configuration, harmonization + def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): """ diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index cbf1a8d1c1..7e09d549de 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -99,4 +99,4 @@ CREATE INDEX "idx_events_source.asn" ON events USING btree ("source.asn"); CREATE INDEX "idx_events_source.ip" ON events USING btree ("source.ip"); CREATE INDEX "idx_events_source.fqdn" ON events USING btree ("source.fqdn"); CREATE INDEX "idx_events_time.observation" ON events USING btree ("time.observation"); -CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); \ No newline at end of file +CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index 6021a0fc0b..a77fff06ca 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -876,6 +876,8 @@ def test_v342_new_fields(self): """ Test adding new harmonisation fields """ result = upgrades.v342_new_fields({}, {"event": {"old-field": "must stay"}}, False) self.assertTrue(result[0]) + self.assertIn("old-field", result[1]["event"]) + self.assertIn("severity", result[1]["event"]) self.assertIn("old-field", result[2]["event"]) self.assertIn("product.full_name", result[2]["event"]) self.assertIn("product.name", result[2]["event"]) @@ -883,6 +885,7 @@ def test_v342_new_fields(self): self.assertIn("product.version", result[2]["event"]) self.assertIn("product.vulnerabilities", result[2]["event"]) + for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name, generate_function(getattr(upgrades, name))) From 2577e6bec149b537445036e4afcdd858fbc7621e Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 3 Apr 2025 17:20:04 +0200 Subject: [PATCH 05/12] Fix style --- intelmq/lib/upgrades.py | 1 + 1 file changed, 1 insertion(+) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index ba8853401c..71fbeecd1d 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -975,6 +975,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): message = f"Found discontinued Twitter collector bot: {', '.join(found_twitter_collector)}" return message or changed, configuration, harmonization + def v341_new_fields(configuration, harmonization, dry_run, **kwargs): changed = False if "severity" not in harmonization["event"]: From 5bcfec7e4178ace3ccaf6fa3c7663814ea5eaa01 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 3 Apr 2025 17:30:57 +0200 Subject: [PATCH 06/12] Fix tests --- intelmq/lib/upgrades.py | 2 +- intelmq/tests/lib/test_upgrades.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 71fbeecd1d..630f3d293c 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -43,7 +43,7 @@ 'v322_removed_feeds_and_bots', 'v340_deprecations', 'v341_blueliv_removal', - 'v342_new_fields' + 'v342_new_fields', ] diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index a77fff06ca..dd71deda84 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -884,6 +884,8 @@ def test_v342_new_fields(self): self.assertIn("product.vendor", result[2]["event"]) self.assertIn("product.version", result[2]["event"]) self.assertIn("product.vulnerabilities", result[2]["event"]) + self.assertIn("old-field", result[2]["event"]) + self.assertIn("severity", result[2]["event"]) for name in upgrades.__all__: From 073358c146a7e732ef9f9c29f60cdeaca7ce9089 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 3 Apr 2025 17:33:23 +0200 Subject: [PATCH 07/12] Add missing docstring --- intelmq/lib/upgrades.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 630f3d293c..934318c957 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -977,6 +977,9 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): def v341_new_fields(configuration, harmonization, dry_run, **kwargs): + """ + Add new fields to IntelMQ Data Format + """ changed = False if "severity" not in harmonization["event"]: harmonization["event"]["severity"] = { From 0528075fa4bb899df09a1e05a5d7ae25f39235f2 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Wed, 9 Apr 2025 16:15:34 +0200 Subject: [PATCH 08/12] Fix handling no event in harmonisation --- intelmq/lib/upgrades.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 934318c957..53e9743cc4 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -981,7 +981,7 @@ def v341_new_fields(configuration, harmonization, dry_run, **kwargs): Add new fields to IntelMQ Data Format """ changed = False - if "severity" not in harmonization["event"]: + if "event" in harmonization and "severity" not in harmonization["event"]: harmonization["event"]["severity"] = { "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", "length": 10, From 8f1a041958531ab8fd221e0983f9cf27b5da7ec1 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Wed, 9 Apr 2025 16:29:02 +0200 Subject: [PATCH 09/12] Simplify and fix upgrade function --- intelmq/lib/upgrades.py | 17 +---------------- intelmq/tests/lib/test_upgrades.py | 2 -- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 53e9743cc4..b2b8b90e13 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -976,22 +976,6 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): return message or changed, configuration, harmonization -def v341_new_fields(configuration, harmonization, dry_run, **kwargs): - """ - Add new fields to IntelMQ Data Format - """ - changed = False - if "event" in harmonization and "severity" not in harmonization["event"]: - harmonization["event"]["severity"] = { - "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", - "length": 10, - "regex": "^(critical|high|medium|low|info|undefined)$", - "type": "LowercaseString", - } - changed = True - return changed, configuration, harmonization - - def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): """ Remove blueliv collector and parser @@ -1027,6 +1011,7 @@ def v342_new_fields(configuration, harmonization, dry_run, **kwargs): resource_filename("intelmq", "etc/harmonization.conf") ) for field in [ + "severity", "product.full_name", "product.name", "product.vendor", diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index dd71deda84..ed2eb1b979 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -876,8 +876,6 @@ def test_v342_new_fields(self): """ Test adding new harmonisation fields """ result = upgrades.v342_new_fields({}, {"event": {"old-field": "must stay"}}, False) self.assertTrue(result[0]) - self.assertIn("old-field", result[1]["event"]) - self.assertIn("severity", result[1]["event"]) self.assertIn("old-field", result[2]["event"]) self.assertIn("product.full_name", result[2]["event"]) self.assertIn("product.name", result[2]["event"]) From 361eb1a88c910ebd3678f07843c903944681520b Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 26 Aug 2025 14:03:24 +0200 Subject: [PATCH 10/12] fix upgrade functions version numbers --- intelmq/lib/upgrades.py | 11 +++++------ intelmq/tests/lib/test_upgrades.py | 14 +++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index b2b8b90e13..208684c033 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -42,8 +42,8 @@ 'v322_url_replacement', 'v322_removed_feeds_and_bots', 'v340_deprecations', - 'v341_blueliv_removal', - 'v342_new_fields', + 'v350_blueliv_removal', + 'v350_new_fields', ] @@ -976,7 +976,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): return message or changed, configuration, harmonization -def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): +def v350_blueliv_removal(configuration, harmonization, dry_run, **kwargs): """ Remove blueliv collector and parser """ @@ -999,7 +999,7 @@ def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): return message, configuration, harmonization -def v342_new_fields(configuration, harmonization, dry_run, **kwargs): +def v350_new_fields(configuration, harmonization, dry_run, **kwargs): """ Add new fields to IntelMQ Data Format """ @@ -1057,8 +1057,7 @@ def v342_new_fields(configuration, harmonization, dry_run, **kwargs): ((3, 3, 0), ()), ((3, 3, 1), ()), ((3, 4, 0), (v340_deprecations, )), - ((3, 4, 1), (v341_blueliv_removal, )), - ((3, 4, 2), (v342_new_fields, )), + ((3, 5, 0), (v350_blueliv_removal, v350_new_fields)), ]) ALWAYS = (harmonization,) diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index ed2eb1b979..047ecb4d43 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -616,7 +616,7 @@ "module": "intelmq.bots.collectors.twitter.collector", }, } -V341_BLUELIV_REMOVAL = { +V350_BLUELIV_REMOVAL = { "global": {}, "blueliv-collector": { "module": "intelmq.bots.collectors.blueliv.collector_crimeserver" @@ -865,16 +865,16 @@ def test_v340_twitter_collector(self): self.assertIn('twitter-collector', result[0]) self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1]) - def test_v341_blueliv_removal(self): - """ Test v341_blueliv_removal deprecation warning """ - result = upgrades.v341_blueliv_removal(V341_BLUELIV_REMOVAL, {}, False) + def test_v350_blueliv_removal(self): + """ Test v350_blueliv_removal deprecation warning """ + result = upgrades.v350_blueliv_removal(V350_BLUELIV_REMOVAL, {}, False) self.assertIn('blueliv-collector', result[0]) self.assertIn('blueliv-parser', result[0]) - self.assertEqual(V341_BLUELIV_REMOVAL, result[1]) + self.assertEqual(V350_BLUELIV_REMOVAL, result[1]) - def test_v342_new_fields(self): + def test_v350_new_fields(self): """ Test adding new harmonisation fields """ - result = upgrades.v342_new_fields({}, {"event": {"old-field": "must stay"}}, False) + result = upgrades.v350_new_fields({}, {"event": {"old-field": "must stay"}}, False) self.assertTrue(result[0]) self.assertIn("old-field", result[2]["event"]) self.assertIn("product.full_name", result[2]["event"]) From 956e8a9aadd787b4c0fa008798dc361531bd6dcd Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 24 Oct 2025 09:47:10 +0200 Subject: [PATCH 11/12] doc: Add severity level explanations --- intelmq/etc/harmonization.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index e25a938720..4cca6c144a 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -254,7 +254,7 @@ "type": "URL" }, "severity": { - "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", + "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source. Allowed values: critical (highly critical vulnerabilities being actively exploited and pose a very high likelihood of compromise. For example, RCEs, sensitive data access), high (end of life systems, accessible internal systems that should not be exposed, risk of data leaks, malware drone and sinkhole events), medium (DDoS-amplifiers, unencrypted services requiring login, vulnerabilities requiring MITM to exploit, attacks need prior knowledge), low (deviation from best practice, little to no practical way to exploit, but setup is not ideal), info (informational only, no known risk), undefined (unknown or undetermined)", "length": 10, "regex": "^(critical|high|medium|low|info|undefined)$", "type": "LowercaseString" From fb975344eaac26e037f35e899f448a2812446134 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 24 Oct 2025 10:48:44 +0200 Subject: [PATCH 12/12] tst: fix debian package installation update first noninteractive apt call --- .github/workflows/debian-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/debian-package.yml b/.github/workflows/debian-package.yml index 9a4d96783c..1877e661bb 100644 --- a/.github/workflows/debian-package.yml +++ b/.github/workflows/debian-package.yml @@ -33,7 +33,7 @@ jobs: run: bash .github/workflows/scripts/debian-package.sh ${{ matrix.codename }} - name: Test packages installation - run: sudo apt install ~/artifacts/*.deb + run: sudo apt-get update && DEBIAN_FRONTEND="noninteractive" sudo apt-get install ~/artifacts/*.deb - name: Upload artifact if: ${{ github.event_name == 'push' }}