From e632b1fa2312a0e6dbbb709da0b7cbda53900e72 Mon Sep 17 00:00:00 2001 From: simpki Date: Fri, 18 May 2018 16:24:10 -0400 Subject: [PATCH 1/5] Add normalization for IPFIX events Uses IPFIX information elements to normalize IPFIX records. --- .../configuration/logstash/netflow.conf.erb | 237 +++++++++++++++++- 1 file changed, 235 insertions(+), 2 deletions(-) diff --git a/modules/netflow/configuration/logstash/netflow.conf.erb b/modules/netflow/configuration/logstash/netflow.conf.erb index 694d2dc244c..4b4a33177e9 100644 --- a/modules/netflow/configuration/logstash/netflow.conf.erb +++ b/modules/netflow/configuration/logstash/netflow.conf.erb @@ -3,7 +3,7 @@ input { type => "netflow" port => <%= setting("var.input.udp.port", 2055) %> codec => netflow { - versions => [5,9] + versions => [5,9,10] } workers => <%= setting("var.input.udp.workers", 2) %> receive_buffer_bytes => <%= setting("var.input.udp.receive_buffer_bytes", 212992) %> @@ -122,7 +122,7 @@ filter { } else { # Did not recognize IP version. mutate { - id => "netflow-ip-version-not-recognized" + id => "IPFIX-ip-version-not-recognized" add_tag => [ "__netflow_ip_version_not_recognized" ] } } @@ -275,6 +275,239 @@ filter { } } + # Process IPFIX events. + else if [netflow][version] == 10 { + mutate { + id => "IPFIX-normalize-version" + replace => { "[netflow][version]" => "IPFIX" } + } + + if [netflow][ipVersion] { + mutate { + id => "IPFIX-normalize-ip_version" + add_field => { "[netflow][ip_version]" => "IPv%{[netflow][ipVersion]}" } + } + } + + # Populate fields with IPv4 or IPv6 specific fields. + if [netflow][sourceIPv4Address] or [netflow][destinationIPv4Address] or [netflow][ipVersion] == 4 { + if [netflow][sourceIPv4Address] { + mutate { + id => "IPFIX-normalize-src_addr-from-sourceIPv4Address" + rename => { "[netflow][sourceIPv4Address]" => "[netflow][src_addr]" } + } + } + if [netflow][sourceIPv4PrefixLength] { + mutate { + id => "IPFIX-normalize-src_mask_len-from-sourceIPv4PrefixLength" + rename => { "[netflow][sourceIPv4PrefixLength]" => "[netflow][src_mask_len]" } + } + } + if [netflow][destinationIPv4Address] { + mutate { + id => "IPFIX-normalize-dst_addr-from-destinationIPv4Address" + rename => { "[netflow][destinationIPv4Address]" => "[netflow][dst_addr]" } + } + } + if [netflow][destinationIPv4PrefixLength] { + mutate { + id => "IPFIX-normalize-dst_mask_len-from-destinationIPv4PrefixLength" + rename => { "[netflow][destinationIPv4PrefixLength]" => "[netflow][dst_mask_len]" } + } + } + if [netflow][ipNextHopIPv4Address] { + mutate { + id => "IPFIX-normalize-next_hop-from-ipNextHopIPv4Address" + rename => { "[netflow][ipNextHopIPv4Address]" => "[netflow][next_hop]" } + } + } + } else if [netflow][sourceIPv6Address] or [netflow][destinationIPv6Address] or [netflow][ipVersion] == 6 { + if [netflow][sourceIPv6Address] { + mutate { + id => "IPFIX-normalize-src_addr-from-sourceIPv6Address" + rename => { "[netflow][sourceIPv6Address]" => "[netflow][src_addr]" } + } + } + if [netflow][sourceIPv6PrefixLength] { + mutate { + id => "IPFIX-normalize-src_mask_len-from-sourceIPv6PrefixLength" + rename => { "[netflow][sourceIPv6PrefixLength]" => "[netflow][src_mask_len]" } + } + } + if [netflow][destinationIPv6Address] { + mutate { + id => "IPFIX-normalize-dst_addr-from-destinationIPv6Address" + rename => { "[netflow][destinationIPv6Address]" => "[netflow][dst_addr]" } + } + } + if [netflow][destinationIPv6PrefixLength] { + mutate { + id => "IPFIX-normalize-dst_mask_len-from-destinationIPv6PrefixLength" + rename => { "[netflow][destinationIPv6PrefixLength]" => "[netflow][dst_mask_len]" } + } + } + if [netflow][ipNextHopIPv6Address] { + mutate { + id => "IPFIX-normalize-next_hop-from-ipNextHopIPv6Address" + rename => { "[netflow][ipNextHopIPv6Address]" => "[netflow][next_hop]" } + } + } + } else { + # Did not recognize IP version. + mutate { + id => "netflow-ip-version-not-recognized" + add_tag => [ "__netflow_ip_version_not_recognized" ] + } + } + + # Populate flow direction (ingress/egress). + if [netflow][flowDirection] == 0 { + mutate { + id => "IPFIX-normalize-flowDirection-ingress" + replace => { "[netflow][flowDirection]" => "ingress" } + } + } else if [netflow][flowDirection] == 1 { + mutate { + id => "IPFIX-normalize-flowDirection-egress" + replace => { "[netflow][flowDirection]" => "egress" } + } + } else { + mutate { + id => "IPFIX-normalize-flowDirection-not-recognized" + add_tag => [ "__netflow_direction_not_recognized" ] + } + } + + # Populate source port. + if [netflow][sourceTransportPort] { + mutate { + id => "IPFIX-normalize-src_port_from_sourceTransportPort" + rename => { "[netflow][sourceTransportPort]" => "[netflow][src_port]" } + } + } else if [netflow][tcpSourcePort] { + mutate { + id => "IPFIX-normalize-src_port_from_tcpSourcePort" + rename => { "[netflow][tcpSourcePort]" => "[netflow][src_port]" } + } + } else if [netflow][udpSourcePort] { + mutate { + id => "IPFIX-normalize-src_port_from_udpSourcePort" + rename => { "[netflow][udpSourcePort]" => "[netflow][src_port]" } + } + } + + # Populate destination port. + if [netflow][destinationTransportPort] { + mutate { + id => "IPFIX-normalize-dst_port_from_destinationTransportPort" + rename => { "[netflow][destinationTransportPort]" => "[netflow][dst_port]" } + } + } else if [netflow][tcpDestinationPort] { + mutate { + id => "IPFIX-normalize-dst_port_from_tcpDestinationPort" + rename => { "[netflow][tcpDestinationPort]" => "[netflow][dst_port]" } + } + } else if [netflow][udpDestinationPort] { + mutate { + id => "IPFIX-normalize-dst_port_from_udpDestinationPort" + rename => { "[netflow][udpDestinationPort]" => "[netflow][dst_port]" } + } + } + + # Populate bytes transferred in the flow. + if [netflow][octetDeltaCount] { + mutate { + id => "IPFIX-normalize-bytes-from-octetDeltaCount" + rename => { "[netflow][octetDeltaCount]" => "[netflow][bytes]" } + } + } else if [netflow][postOctetDeltaCount] { + mutate { + id => "IPFIX-normalize-bytes-from-postOctetDeltaCount" + rename => { "[netflow][postOctetDeltaCount]" => "[netflow][bytes]" } + } + } else if [netflow][octetTotalCount] { + mutate { + id => "IPFIX-normalize-bytes-from-octetTotalCount" + rename => { "[netflow][octetTotalCount]" => "[netflow][bytes]" } + } + } + if [netflow][bytes] { + mutate { + id => "IPFIX-normalize-convert-bytes" + convert => { "[netflow][bytes]" => "integer" } + } + } + + # Populate packets transferred in the flow. + if [netflow][packetDeltaCount] { + mutate { + id => "IPFIX-normalize-packets-from-packetDeltaCount" + rename => { "[netflow][packetDeltaCount]" => "[netflow][packets]" } + } + } else if [netflow][postPacketDeltaCount] { + mutate { + id => "IPFIX-normalize-packets-from-postPacketDeltaCount" + rename => { "[netflow][postPacketDeltaCount]" => "[netflow][packets]" } + } + } else if [netflow][packetTotalCount] { + mutate { + id => "IPFIX-normalize-packets-from-packetTotalCount" + rename => { "[netflow][packetTotalCount]" => "[netflow][packets]" } + } + } + if [netflow][packets] { + mutate { + id => "IPFIX-normalize-convert-packets" + convert => { "[netflow][packets]" => "integer" } + } + } + + # Populate source and destination MAC addresses if available. + if [netflow][sourceMacAddress] { + mutate { + id => "IPFIX-normalize-src_mac-from-sourceMacAddress" + rename => { "[netflow][sourceMacAddress]" => "[netflow][src_mac]" } + } + } else if [netflow][postSourceMacAddress] { + mutate { + id => "IPFIX-normalize-src_mac-from-postSourceMacAddress" + rename => { "[netflow][postSourceMacAddress]" => "[netflow][src_mac]" } + } + } + if [netflow][destinationMacAddress] { + mutate { + id => "IPFIX-normalize-dst_mac-from-destinationMacAddress" + rename => { "[netflow][destinationMacAddress]" => "[netflow][dst_mac]" } + } + } else if [netflow][postDestinationMacAddress] { + mutate { + id => "IPFIX-normalize-dst_mac-from-postDestinationMacAddress" + rename => { "[netflow][postDestinationMacAddress]" => "[netflow][dst_mac]" } + } + } + + # Populate VLAN if available. + if [netflow][vlanId] { + mutate { + id => "IPFIX-normalize-vlan-from-vlanId" + rename => { "[netflow][vlanId]" => "[netflow][vlan]" } + } + } else if [netflow][postVlanId] { + mutate { + id => "IPFIX-normalize-vlan-from-postVlanId" + rename => { "[netflow][postVlanId]" => "[netflow][vlan]" } + } + } + + # Populate TOS if available. + if [netflow][ipClassOfService] { + mutate { + id => "IPFIX-normalize-tos-from-ipClassOfService" + rename => { "[netflow][ipClassOfService]" => "[netflow][tos]" } + } + } + } + #-------------------- # We now have a normalized flow record. The rest of the logic works # regardless of the Netflow version. From 4f57401d8ff8febac2ee265803289d51c9c8c5cb Mon Sep 17 00:00:00 2001 From: simpki Date: Tue, 22 May 2018 15:02:08 -0400 Subject: [PATCH 2/5] add protocol normalization --- modules/netflow/configuration/logstash/netflow.conf.erb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/netflow/configuration/logstash/netflow.conf.erb b/modules/netflow/configuration/logstash/netflow.conf.erb index 4b4a33177e9..780839aecc8 100644 --- a/modules/netflow/configuration/logstash/netflow.conf.erb +++ b/modules/netflow/configuration/logstash/netflow.conf.erb @@ -506,6 +506,14 @@ filter { rename => { "[netflow][ipClassOfService]" => "[netflow][tos]" } } } + + # Populate protocol if available. + if [netflow][protocolIdentifier] { + mutate { + id => "IPFIX-normalize-protocol-from-protocolIdentifier" + rename => { "[netflow][protocolIdentifier]" => "[netflow][protocol]" } + } + } } #-------------------- From b8a36dbc812870734ca1e63f8827af78d921075b Mon Sep 17 00:00:00 2001 From: simpki Date: Tue, 22 May 2018 15:10:34 -0400 Subject: [PATCH 3/5] fix misordered ipversion parameter --- modules/netflow/configuration/logstash/netflow.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/netflow/configuration/logstash/netflow.conf.erb b/modules/netflow/configuration/logstash/netflow.conf.erb index 780839aecc8..4e64812f28a 100644 --- a/modules/netflow/configuration/logstash/netflow.conf.erb +++ b/modules/netflow/configuration/logstash/netflow.conf.erb @@ -285,7 +285,7 @@ filter { if [netflow][ipVersion] { mutate { id => "IPFIX-normalize-ip_version" - add_field => { "[netflow][ip_version]" => "IPv%{[netflow][ipVersion]}" } + add_field => { "[netflow][ipVersion]" => "IPv%{[netflow][ip_version]}" } } } From 2019b1a1b68e1f09d3a45c263f1f4b487116cc3f Mon Sep 17 00:00:00 2001 From: simpki Date: Tue, 22 May 2018 15:15:30 -0400 Subject: [PATCH 4/5] revert --- modules/netflow/configuration/logstash/netflow.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/netflow/configuration/logstash/netflow.conf.erb b/modules/netflow/configuration/logstash/netflow.conf.erb index 4e64812f28a..780839aecc8 100644 --- a/modules/netflow/configuration/logstash/netflow.conf.erb +++ b/modules/netflow/configuration/logstash/netflow.conf.erb @@ -285,7 +285,7 @@ filter { if [netflow][ipVersion] { mutate { id => "IPFIX-normalize-ip_version" - add_field => { "[netflow][ipVersion]" => "IPv%{[netflow][ip_version]}" } + add_field => { "[netflow][ip_version]" => "IPv%{[netflow][ipVersion]}" } } } From d07928e061cd7ff15687f1c879fc0b07672f2450 Mon Sep 17 00:00:00 2001 From: simpki Date: Tue, 22 May 2018 16:44:46 -0400 Subject: [PATCH 5/5] add tcp flags --- modules/netflow/configuration/logstash/netflow.conf.erb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/netflow/configuration/logstash/netflow.conf.erb b/modules/netflow/configuration/logstash/netflow.conf.erb index 780839aecc8..3d9e45809f0 100644 --- a/modules/netflow/configuration/logstash/netflow.conf.erb +++ b/modules/netflow/configuration/logstash/netflow.conf.erb @@ -514,6 +514,14 @@ filter { rename => { "[netflow][protocolIdentifier]" => "[netflow][protocol]" } } } + + # Populate TCP flags if available. + if [netflow][tcpControlBits] { + mutate { + id => "IPFIX-normalize-tcp_flags-from-tcpControlBits" + rename => { "[netflow][tcpControlBits]" => "[netflow][tcp_flags]" } + } + } } #--------------------