Extended simple-nva module to manage BGP service running on FR routing docker container

modules/cloud-config-container/simple-nva/README.md

@@ -68,10 +68,12 @@ module "vm" {
 
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [network_interfaces](variables.tf#L39) | Network interfaces configuration. | <code title="list&#40;object&#40;&#123;&#10;  routes &#61; optional&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;&#41;">list&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | ✓ |  |
-| [cloud_config](variables.tf#L17) | Cloud config template path. If null default will be used. | <code>string</code> |  | <code>null</code> |
-| [enable_health_checks](variables.tf#L23) | Configures routing to enable responses to health check probes. | <code>bool</code> |  | <code>false</code> |
-| [files](variables.tf#L29) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | <code title="map&#40;object&#40;&#123;&#10;  content     &#61; string&#10;  owner       &#61; string&#10;  permissions &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [network_interfaces](variables.tf#L51) | Network interfaces configuration. | <code title="list&#40;object&#40;&#123;&#10;  routes              &#61; optional&#40;list&#40;string&#41;&#41;&#10;  enable_masquerading &#61; optional&#40;bool&#41;&#10;  non_masq_cidrs      &#61; optional&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;&#41;">list&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | ✓ |  |
+| [bgp_config](variables.tf#L17) | BGP configuration for FR Routing container running on the NVA. | <code title="object&#40;&#123;&#10;  daemons    &#61; optional&#40;string&#41;&#10;  enable     &#61; optional&#40;bool&#41;&#10;  frr_config &#61; optional&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  enable &#61; false&#10;&#125;">&#123;&#8230;&#125;</code> |
+| [cloud_config](variables.tf#L29) | Cloud config template path. If null default will be used. | <code>string</code> |  | <code>null</code> |
+| [enable_health_checks](variables.tf#L35) | Configures routing to enable responses to health check probes. | <code>bool</code> |  | <code>false</code> |
+| [files](variables.tf#L41) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | <code title="map&#40;object&#40;&#123;&#10;  content     &#61; string&#10;  owner       &#61; string&#10;  permissions &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [optional_run_cmds](variables.tf#L60) | Optional Cloud Init run commands to execute. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
 
 ## Outputs
 

modules/cloud-config-container/simple-nva/cloud-config.yaml

@@ -43,6 +43,12 @@ write_files:
 %{ if enable_health_checks ~}
       /var/run/nva/policy_based_routing.sh ${interface.name}
 %{ endif ~}
+%{ if interface.enable_masquerading ~}
+%{ for cidr in interface.non_masq_cidrs ~}
+      iptables -t nat -A POSTROUTING -o ${interface.name} -d ${cidr} -j ACCEPT
+%{ endfor ~}
+      iptables -t nat -A POSTROUTING -o ${interface.name} -j MASQUERADE
+%{ endif ~}
 %{ for route in interface.routes ~}
       ip route add ${route} via `curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/${interface.number}/gateway -H "Metadata-Flavor:Google"` dev ${interface.name}
 %{ endfor ~}
@@ -55,4 +61,6 @@ runcmd:
   - systemctl daemon-reload
   - systemctl enable routing
   - systemctl start routing
-
+%{ for cmd in optional_run_cmds ~}
+  - ${cmd}
+%{ endfor ~}

modules/cloud-config-container/simple-nva/files/frr/daemons

@@ -0,0 +1,66 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+zebra=no
+bgpd=yes
+ospfd=no
+ospf6d=no
+ripd=no
+ripngd=no
+isisd=no
+pimd=no
+ldpd=no
+nhrpd=no
+eigrpd=no
+babeld=no
+sharpd=no
+staticd=no
+pbrd=no
+bfdd=no
+fabricd=no
+
+#
+# If this option is set the /etc/init.d/frr script automatically loads
+# the config via "vtysh -b" when the servers are started.
+# Check /etc/pam.d/frr if you intend to use "vtysh"!
+#
+vtysh_enable=yes
+zebra_options="  -A 127.0.0.1 -s 90000000"
+bgpd_options="   -A 127.0.0.1"
+ospfd_options="  --daemon -A 127.0.0.1"
+ospf6d_options=" --daemon -A ::1"
+ripd_options="   --daemon -A 127.0.0.1"
+ripngd_options=" --daemon -A ::1"
+isisd_options="  --daemon -A 127.0.0.1"
+pimd_options="  --daemon -A 127.0.0.1"
+ldpd_options="  --daemon -A 127.0.0.1"
+nhrpd_options="  --daemon -A 127.0.0.1"
+eigrpd_options="  --daemon -A 127.0.0.1"
+babeld_options="  --daemon -A 127.0.0.1"
+sharpd_options="  --daemon -A 127.0.0.1"
+staticd_options="  --daemon -A 127.0.0.1"
+pbrd_options="  --daemon -A 127.0.0.1"
+bfdd_options="  --daemon -A 127.0.0.1"
+fabricd_options="  --daemon -A 127.0.0.1"
+
+#MAX_FDS=1024
+# The list of daemons to watch is automatically generated by the init script.
+#watchfrr_options=""
+
+# for debugging purposes, you can specify a "wrap" command to start instead
+# of starting the daemon directly, e.g. to use valgrind on ospfd:
+#   ospfd_wrap="/usr/bin/valgrind"
+# or you can use "all_wrap" for all daemons, e.g. to use perf record:
+#   all_wrap="/usr/bin/perf record --call-graph -"
+# the normal daemon command is added to this at the end.

modules/cloud-config-container/simple-nva/files/frr/frr.conf

@@ -0,0 +1,23 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default FRR configuration which enables a BGP protocol process with a default ASN of 65500.
+# For more information on how to update this file and/or configure bgp sessions please refer to 
+# the official documentation available at the following link:
+# https://docs.frrouting.org/en/latest/overview.html
+
+log syslog informational
+no ipv6 forwarding
+router bgp 65500
+line vty

modules/cloud-config-container/simple-nva/files/frr/frr.service

@@ -0,0 +1,27 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[Unit]
+Description=Start FRR container
+After=gcr-online.target docker.socket
+Wants=gcr-online.target docker.socket docker-events-collector.service
+[Service]
+Environment="HOME=/home/frr"
+ExecStart=/usr/bin/docker run --rm --name=frr \
+--privileged \
+--network host \
+-v /etc/frr:/etc/frr \
+frrouting/frr
+ExecStop=/usr/bin/docker stop frr
+ExecStopPost=/usr/bin/docker rm frr

modules/cloud-config-container/simple-nva/files/policy_based_routing.sh

@@ -19,8 +19,8 @@ IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ")
 
 # Sleep while there's no load balancer IP route for this IF
 while [ -z $IP_LB ] ; do
-  sleep 2
-  IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ")
+   sleep 2
+   IP_LB=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " ")
 done
 
 IF_NUMBER=$(echo $IF_NAME | sed -e s/eth//)
@@ -31,4 +31,4 @@ IF_IP_PREFIX=$(/var/run/nva/ipprefix_by_netmask.sh $IF_NETMASK)
 grep -qxF "$((200 + $IF_NUMBER)) hc-$IF_NAME" /etc/iproute2/rt_tables || echo "$((200 + $IF_NUMBER)) hc-$IF_NAME" >>/etc/iproute2/rt_tables
 ip route add $IF_GW src $IF_IP dev $IF_NAME table hc-$IF_NAME
 ip route add default via $IF_GW dev $IF_NAME table hc-$IF_NAME
-ip rule add from $IP_LB/32 table hc-$IF_NAME
+ip rule add from $IP_LB/32 table hc-$IF_NAME

modules/cloud-config-container/simple-nva/main.tf

@@ -19,35 +19,80 @@ locals {
     files                = local.files
     enable_health_checks = var.enable_health_checks
     network_interfaces   = local.network_interfaces
+    optional_run_cmds    = local.optional_run_cmds
   }))
 
-  files = merge({
-    "/var/run/nva/ipprefix_by_netmask.sh" = {
-      content     = file("${path.module}/files/ipprefix_by_netmask.sh")
-      owner       = "root"
-      permissions = "0744"
-    }
-    "/var/run/nva/policy_based_routing.sh" = {
-      content     = file("${path.module}/files/policy_based_routing.sh")
-      owner       = "root"
-      permissions = "0744"
-    }
-    }, {
-    for path, attrs in var.files : path => {
-      content     = attrs.content,
-      owner       = attrs.owner,
-      permissions = attrs.permissions
-    }
-  })
+  frr_config = (
+    try(var.bgp_config.frr_config != null, false) ? var.bgp_config.frr_config : "${path.module}/files/frr/frr.conf"
+  )
+  daemons = (
+    try(var.bgp_config.daemons != null, false) ? var.bgp_config.daemons : "${path.module}/files/frr/daemons"
+  )
+  files = merge(
+    {
+      "/var/run/nva/ipprefix_by_netmask.sh" = {
+        content     = file("${path.module}/files/ipprefix_by_netmask.sh")
+        owner       = "root"
+        permissions = "0744"
+      }
+      "/var/run/nva/policy_based_routing.sh" = {
+        content     = file("${path.module}/files/policy_based_routing.sh")
+        owner       = "root"
+        permissions = "0744"
+      }
+      }, {
+      for path, attrs in var.files : path => {
+        content     = attrs.content,
+        owner       = attrs.owner,
+        permissions = attrs.permissions
+      }
+    },
+    try(var.bgp_config.enable, false) ? {
+      "/etc/frr/daemons" = {
+        content     = file(local.daemons)
+        owner       = "root"
+        permissions = "0744"
+      }
+      "/etc/frr/frr.conf" = {
+        content     = file(local.frr_config)
+        owner       = "root"
+        permissions = "0744"
+      }
+      "/etc/systemd/system/frr.service" = {
+        content     = file("${path.module}/files/frr/frr.service")
+        owner       = "root"
+        permissions = "0644"
+      }
+      "/var/lib/docker/daemon.json" = {
+        content     = <<EOF
+        {
+          "live-restore": true,
+          "storage-driver": "overlay2",
+          "log-opts": {
+            "max-size": "1024m"
+          }
+        }
+        EOF
+        owner       = "root"
+        permissions = "0644"
+      }
+    } : {}
+  )
 
   network_interfaces = [
     for index, interface in var.network_interfaces : {
-      name   = "eth${index}"
-      number = index
-      routes = interface.routes
+      name                = "eth${index}"
+      number              = index
+      routes              = interface.routes
+      enable_masquerading = interface.enable_masquerading != null ? interface.enable_masquerading : false
+      non_masq_cidrs      = interface.non_masq_cidrs != null ? interface.non_masq_cidrs : []
     }
   ]
 
+  optional_run_cmds = try(var.bgp_config.enable, false) ? concat(
+    ["systemctl start frr"], var.optional_run_cmds
+  ) : var.optional_run_cmds
+
   template = (
     var.cloud_config == null
     ? "${path.module}/cloud-config.yaml"

modules/cloud-config-container/simple-nva/variables.tf

@@ -14,6 +14,18 @@
  * limitations under the License.
  */
 
+variable "bgp_config" {
+  description = "BGP configuration for FR Routing container running on the NVA."
+  type = object({
+    daemons    = optional(string)
+    enable     = optional(bool)
+    frr_config = optional(string)
+  })
+  default = {
+    enable = false
+  }
+}
+
 variable "cloud_config" {
   description = "Cloud config template path. If null default will be used."
   type        = string
@@ -39,6 +51,14 @@ variable "files" {
 variable "network_interfaces" {
   description = "Network interfaces configuration."
   type = list(object({
-    routes = optional(list(string))
+    routes              = optional(list(string))
+    enable_masquerading = optional(bool)
+    non_masq_cidrs      = optional(list(string))
   }))
 }
+
+variable "optional_run_cmds" {
+  description = "Optional Cloud Init run commands to execute."
+  type        = list(string)
+  default     = []
+}