diff --git a/packages/suricata/changelog.yml b/packages/suricata/changelog.yml index 0c78abcdd69..344dbd0d72b 100644 --- a/packages/suricata/changelog.yml +++ b/packages/suricata/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.18.0" + changes: + - description: Populate `observer.*` fields if "forwarded" in tags + type: enhancement + link: https://github.com/elastic/integrations/pull/6985 - version: "2.17.1" changes: - description: Refer to the ECS definition for all fields that exist in ECS. diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-alerts.log-config.yml b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-alerts.log-config.yml new file mode 100755 index 00000000000..462470cdfbd --- /dev/null +++ b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-alerts.log-config.yml @@ -0,0 +1,4 @@ +fields: + tags: + - forwarded + - preserve_original_event diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-alerts.log-expected.json b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-alerts.log-expected.json index b4af5b0e8fa..9d785ac7367 100644 --- a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-alerts.log-expected.json +++ b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-alerts.log-expected.json @@ -64,6 +64,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "example.net" @@ -106,6 +111,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -186,6 +192,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "example.net" @@ -228,6 +239,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -308,6 +320,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "example.net" @@ -350,6 +367,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -430,6 +448,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "example.org" @@ -472,6 +495,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -552,6 +576,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "example.org" @@ -594,6 +623,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -674,6 +704,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "example.org" @@ -716,6 +751,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -796,6 +832,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "security.ubuntu.com" @@ -837,6 +878,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -920,6 +962,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "archive.ubuntu.com" @@ -961,6 +1008,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -1044,6 +1092,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "archive.ubuntu.com" @@ -1085,6 +1138,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -1168,6 +1222,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "security.ubuntu.com" @@ -1209,6 +1268,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -1292,6 +1352,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "security.ubuntu.com" @@ -1333,6 +1398,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -1416,6 +1482,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "security.ubuntu.com" @@ -1457,6 +1528,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -1540,6 +1612,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "archive.ubuntu.com" @@ -1581,6 +1658,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -1664,6 +1742,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "archive.ubuntu.com" @@ -1705,6 +1788,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -1788,6 +1872,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "archive.ubuntu.com" @@ -1829,6 +1918,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -1912,6 +2002,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "archive.ubuntu.com" @@ -1953,6 +2048,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -2036,6 +2132,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "archive.ubuntu.com" @@ -2077,6 +2178,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -2160,6 +2262,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "archive.ubuntu.com" @@ -2201,6 +2308,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -2283,6 +2391,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "archive.ubuntu.com" @@ -2324,6 +2437,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -2406,6 +2520,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "archive.ubuntu.com" @@ -2447,6 +2566,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -2493,6 +2613,11 @@ "protocol": "tls", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hash": [ "00112233445566778899AABBCCDDEEFF00112233", @@ -2538,6 +2663,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "tls": { @@ -2627,6 +2753,11 @@ "protocol": "tls", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hash": [ "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC", @@ -2686,6 +2817,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "tls": { diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-small.log-config.yml b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-small.log-config.yml new file mode 100755 index 00000000000..462470cdfbd --- /dev/null +++ b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-small.log-config.yml @@ -0,0 +1,4 @@ +fields: + tags: + - forwarded + - preserve_original_event diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-small.log-expected.json b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-small.log-expected.json index 6c0a2e1e22b..2541715ba4f 100644 --- a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-small.log-expected.json +++ b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-small.log-expected.json @@ -26,6 +26,11 @@ "protocol": "ssh", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "ip": [ "192.168.86.85", @@ -55,6 +60,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ] }, @@ -93,6 +99,11 @@ "protocol": "tls", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "l2.io" @@ -136,6 +147,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "tls": { @@ -188,6 +200,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "192.168.86.28" @@ -215,6 +232,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -275,6 +293,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "192.168.86.28" @@ -310,6 +333,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -378,6 +402,11 @@ "protocol": "dns", "transport": "udp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "ip": [ "175.16.199.1", @@ -407,6 +436,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ] }, @@ -423,6 +453,11 @@ "kind": "metric", "original": "{\"timestamp\":\"2018-07-05T15:51:23.009510-0400\",\"event_type\":\"stats\",\"stats\":{\"uptime\":5400,\"capture\":{\"kernel_packets\":430313,\"kernel_drops\":0,\"kernel_ifdrops\":0},\"decoder\":{\"pkts\":430313,\"bytes\":335138381,\"invalid\":2,\"ipv4\":425873,\"ipv6\":3785,\"ethernet\":430313,\"raw\":0,\"null\":0,\"sll\":0,\"tcp\":370093,\"udp\":58337,\"sctp\":0,\"icmpv4\":186,\"icmpv6\":1019,\"ppp\":0,\"pppoe\":0,\"gre\":0,\"vlan\":0,\"vlan_qinq\":0,\"ieee8021ah\":0,\"teredo\":1,\"ipv4_in_ipv6\":0,\"ipv6_in_ipv6\":0,\"mpls\":0,\"avg_pkt_size\":778,\"max_pkt_size\":1514,\"erspan\":0,\"ipraw\":{\"invalid_ip_version\":0},\"ltnull\":{\"pkt_too_small\":0,\"unsupported_type\":0},\"dce\":{\"pkt_too_small\":0}},\"flow\":{\"memcap\":0,\"tcp\":1113,\"udp\":1881,\"icmpv4\":0,\"icmpv6\":677,\"spare\":10000,\"emerg_mode_entered\":0,\"emerg_mode_over\":0,\"tcp_reuse\":0,\"memuse\":11537312},\"defrag\":{\"ipv4\":{\"fragments\":0,\"reassembled\":0,\"timeouts\":0},\"ipv6\":{\"fragments\":0,\"reassembled\":0,\"timeouts\":0},\"max_frag_hits\":0},\"tcp\":{\"sessions\":842,\"ssn_memcap_drop\":0,\"pseudo\":0,\"pseudo_failed\":0,\"invalid_checksum\":0,\"no_flow\":0,\"syn\":1138,\"synack\":656,\"rst\":1165,\"segment_memcap_drop\":0,\"stream_depth_reached\":63,\"reassembly_gap\":0,\"overlap\":5979,\"overlap_diff_data\":0,\"insert_data_normal_fail\":0,\"insert_data_overlap_fail\":0,\"insert_list_fail\":0,\"memuse\":4587520,\"reassembly_memuse\":768000},\"detect\":{\"alert\":2},\"app_layer\":{\"flow\":{\"http\":22,\"ftp\":0,\"smtp\":0,\"tls\":560,\"ssh\":4,\"imap\":0,\"msn\":0,\"smb\":0,\"dcerpc_tcp\":0,\"dns_tcp\":0,\"failed_tcp\":2,\"dcerpc_udp\":0,\"dns_udp\":762,\"failed_udp\":1119},\"tx\":{\"http\":25,\"ftp\":0,\"smtp\":0,\"tls\":0,\"ssh\":0,\"smb\":0,\"dcerpc_tcp\":0,\"dns_tcp\":0,\"dcerpc_udp\":0,\"dns_udp\":762}},\"flow_mgr\":{\"closed_pruned\":729,\"new_pruned\":1879,\"est_pruned\":975,\"bypassed_pruned\":0,\"flows_checked\":8,\"flows_notimeout\":8,\"flows_timeout\":0,\"flows_timeout_inuse\":0,\"flows_removed\":0,\"rows_checked\":65536,\"rows_skipped\":65530,\"rows_empty\":0,\"rows_busy\":0,\"rows_maxlen\":2},\"file_store\":{\"open_files\":0},\"dns\":{\"memuse\":7749,\"memcap_state\":0,\"memcap_global\":0},\"http\":{\"memuse\":17861,\"memcap\":0}}}" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "suricata": { "eve": { "event_type": "stats", @@ -583,6 +618,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ] }, @@ -631,6 +667,11 @@ "protocol": "tls", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hash": [ "6AFFACA65F8A05E7A98C7629B908C769ADDC7247" @@ -666,6 +707,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "tls": { @@ -754,6 +796,11 @@ "packets": 1, "transport": "udp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "ip": [ "fe80:0000:0000:0000:fada:0cff:fedc:87f1", @@ -780,6 +827,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ] }, @@ -822,6 +870,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "ctldl.windowsupdate.com" @@ -847,6 +900,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -890,6 +944,11 @@ "protocol": "tls", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hash": [ "183C114546E926C787640FED47861B31BF0F8425", @@ -926,6 +985,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "tls": { @@ -1018,6 +1078,11 @@ "protocol": "http", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hosts": [ "192.168.50.1" @@ -1044,6 +1109,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "url": { @@ -1092,6 +1158,11 @@ "protocol": "tls", "transport": "tcp" }, + "observer": { + "product": "Suricata", + "type": "ids", + "vendor": "OISF" + }, "related": { "hash": [ "44d502d471cfdb99c59bdfb0f220e5a8" @@ -1125,6 +1196,7 @@ } }, "tags": [ + "forwarded", "preserve_original_event" ], "tls": { diff --git a/packages/suricata/data_stream/eve/agent/stream/log.yml.hbs b/packages/suricata/data_stream/eve/agent/stream/log.yml.hbs index 0acdce56155..ffefd3cd48c 100644 --- a/packages/suricata/data_stream/eve/agent/stream/log.yml.hbs +++ b/packages/suricata/data_stream/eve/agent/stream/log.yml.hbs @@ -13,7 +13,12 @@ tags: {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true {{/contains}} -{{#if processors}} processors: +{{#contains "forwarded" tags}} +- add_observer_metadata: ~ +{{else}} +- add_host_metadata: ~ +{{/contains}} +{{#if processors}} {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml index 55479b741b1..80dd50cdb3f 100644 --- a/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml +++ b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml @@ -5,6 +5,18 @@ processors: - set: field: ecs.version value: '8.10.0' + - set: + if: ctx.tags != null && ctx.tags.contains('forwarded') + field: observer.product + value: Suricata + - set: + if: ctx.tags != null && ctx.tags.contains('forwarded') + field: observer.type + value: ids + - set: + if: ctx.tags != null && ctx.tags.contains('forwarded') + field: observer.vendor + value: OISF - set: field: event.original copy_from: message diff --git a/packages/suricata/data_stream/eve/fields/ecs.yml b/packages/suricata/data_stream/eve/fields/ecs.yml index cc03cf2e71a..8a15a0f4d2d 100644 --- a/packages/suricata/data_stream/eve/fields/ecs.yml +++ b/packages/suricata/data_stream/eve/fields/ecs.yml @@ -114,6 +114,20 @@ name: network.protocol - external: ecs name: network.transport +- external: ecs + name: observer.hostname +- external: ecs + name: observer.ip +- external: ecs + name: observer.mac +- external: ecs + name: observer.name +- external: ecs + name: observer.product +- external: ecs + name: observer.type +- external: ecs + name: observer.vendor - external: ecs name: related.hash - external: ecs diff --git a/packages/suricata/docs/README.md b/packages/suricata/docs/README.md index e4c5ea623cd..69c90096e75 100644 --- a/packages/suricata/docs/README.md +++ b/packages/suricata/docs/README.md @@ -204,6 +204,13 @@ An example event for `eve` looks as following: | network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | | network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | +| observer.hostname | Hostname of the observer. | keyword | +| observer.ip | IP addresses of the observer. | ip | +| observer.mac | MAC addresses of the observer. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | +| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | +| observer.product | The product name of the observer. | keyword | +| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | +| observer.vendor | Vendor name of the observer. | keyword | | related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | | related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | | related.ip | All of the IPs seen on your event. | ip | diff --git a/packages/suricata/manifest.yml b/packages/suricata/manifest.yml index 71bdd719fa5..603337f81de 100644 --- a/packages/suricata/manifest.yml +++ b/packages/suricata/manifest.yml @@ -1,6 +1,6 @@ name: suricata title: Suricata -version: "2.17.1" +version: "2.18.0" description: Collect logs from Suricata with Elastic Agent. type: integration icons: