SONiC E2E: provision NetBox with docker compose

.gitignore

@@ -1,4 +1,5 @@
 *.egg-info
+.venv-sonic-e2e/
 *.pyc
 *.swp
 __pycache__

.zuul.yaml

@@ -140,6 +140,34 @@
     pre-run: playbooks/pre.yml
     run: playbooks/test-integration.yml
 
+# End-to-end golden test for the SONiC config generator: provisions
+# NetBox with a docker compose stack, seeds it with the netbox-manager
+# example data, runs sync_sonic() and compares the exported config_db files
+# against tests/e2e/golden/. In check the job only runs when files that can
+# change the generated output (or the harness itself) are touched;
+# periodic-daily runs it unconditionally and catches drift of the
+# netbox-manager seed data, which is consumed at tip-of-main via
+# required-projects (honoring Depends-On).
+- job:
+    name: python-osism-sonic-e2e
+    nodeset: ubuntu-noble
+    pre-run: playbooks/pre-sonic-e2e.yml
+    run: playbooks/test-sonic-e2e.yml
+    required-projects:
+      - osism/netbox-manager
+    timeout: 2400
+    files:
+      - ^\.zuul\.yaml$
+      - ^Makefile$
+      - ^Pipfile\.lock$
+      - ^files/sonic/.*
+      - ^osism/settings\.py$
+      - ^osism/tasks/conductor/.*
+      - ^osism/utils/.*
+      - ^playbooks/(pre-|test-)sonic-e2e\.yml$
+      - ^requirements\.txt$
+      - ^tests/e2e/.*
+
 - project:
     merge-mode: squash-merge
     default-branch: main
@@ -154,6 +182,7 @@
         - python-osism-test-setup
         - python-osism-unit-tests
         - python-osism-integration-tests
+        - python-osism-sonic-e2e
     periodic-daily:
       jobs:
         - flake8
@@ -163,6 +192,7 @@
         - python-osism-test-setup
         - python-osism-unit-tests
         - python-osism-integration-tests
+        - python-osism-sonic-e2e
     periodic-midnight:
       jobs:
         - container-image-python-osism-push

Makefile

@@ -0,0 +1,24 @@
+NETBOX_MANAGER_DIR ?= $(abspath ../netbox-manager)
+
+# SONiC config-generation E2E golden test (see tests/e2e/sonic_golden_test.sh).
+
+# Full cycle: start the NetBox compose stack (an existing stack is reused
+# and left in place), seed, generate, compare against tests/e2e/golden/.
+sonic-e2e:
+	NETBOX_MANAGER_DIR=$(NETBOX_MANAGER_DIR) tests/e2e/sonic_golden_test.sh
+
+# Regenerate the golden files after an intentional generator change,
+# then review and commit the diff.
+sonic-e2e-regen:
+	NETBOX_MANAGER_DIR=$(NETBOX_MANAGER_DIR) tests/e2e/sonic_golden_test.sh --regenerate
+
+# Start the NetBox stack and leave it running for debugging. Export a
+# NETBOX_TOKEN beforehand to get a known API token minted.
+sonic-e2e-up:
+	tests/e2e/deploy_netbox.sh
+
+# Stop the NetBox stack and remove its volumes.
+sonic-e2e-down:
+	docker compose -f tests/e2e/compose.yaml down --volumes --remove-orphans
+
+.PHONY: sonic-e2e sonic-e2e-regen sonic-e2e-up sonic-e2e-down

osism/settings.py

@@ -70,6 +70,10 @@ def read_secret(secret_name):
 SONIC_EXPORT_SUFFIX = os.getenv("SONIC_EXPORT_SUFFIX", "_config_db.json")
 SONIC_EXPORT_IDENTIFIER = os.getenv("SONIC_EXPORT_IDENTIFIER", "serial-number")
 
+# Directory holding the per-HWSKU port_config .ini files (bundled in the
+# repo under files/sonic/port_config and installed by the Dockerfile)
+SONIC_PORT_CONFIG_PATH = os.getenv("SONIC_PORT_CONFIG_PATH", "/etc/sonic/port_config")
+
 # SONiC ZTP firmware configuration
 #
 # The ZTP firmware install uses a dynamic-url built from

osism/tasks/conductor/sonic/config_generator.py

@@ -508,7 +508,11 @@ def generate_sonic_config(device, hwsku, device_as_mapping=None, config_version=
     if breakout_info["breakout_cfgs"]:
         config["BREAKOUT_CFG"].update(breakout_info["breakout_cfgs"])
     if breakout_info["breakout_ports"]:
-        config["BREAKOUT_PORTS"].update(breakout_info["breakout_ports"])
+        # Project each entry to the SONiC schema ({port: {"master": <port>}});
+        # the full stash (declared/lanes/speed) stays in breakout_info for the
+        # PORT-building helpers and must not leak into the owned table.
+        for child, entry in breakout_info["breakout_ports"].items():
+            config["BREAKOUT_PORTS"][child] = {"master": entry["master"]}
 
     # Add port channel configuration
     _add_portchannel_configuration(config, portchannel_info)
@@ -609,33 +613,45 @@ def _add_port_configurations(
                     port_speed = sonic_speed
 
         if port_name in breakout_info["breakout_ports"]:
+            bp = breakout_info["breakout_ports"][port_name]
             # Get the master port to determine original speed and lanes
-            master_port = breakout_info["breakout_ports"][port_name]["master"]
+            master_port = bp["master"]
 
-            # Override with individual breakout port speed from NetBox if available
-            if port_name in netbox_interfaces and netbox_interfaces[port_name]["speed"]:
-                port_speed = str(int(netbox_interfaces[port_name]["speed"]))
-                logger.debug(
-                    f"Using NetBox speed {port_speed} Mbps for breakout port {port_name}"
+            if bp.get("declared"):
+                # Declared-mode stash is authoritative: it takes precedence over
+                # the NetBox-speed override and the inferred lane calculation.
+                port_speed = str(bp["speed"])
+                port_lanes = bp["lanes"]
+            else:
+                # Override with individual breakout port speed from NetBox if available
+                if (
+                    port_name in netbox_interfaces
+                    and netbox_interfaces[port_name]["speed"]
+                ):
+                    port_speed = str(int(netbox_interfaces[port_name]["speed"]))
+                    logger.debug(
+                        f"Using NetBox speed {port_speed} Mbps for breakout port {port_name}"
+                    )
+                elif master_port in breakout_info["breakout_cfgs"]:
+                    # Fallback to extracting speed from breakout mode
+                    brkout_mode = breakout_info["breakout_cfgs"][master_port][
+                        "brkout_mode"
+                    ]
+                    if "10G" in brkout_mode:
+                        port_speed = "10000"
+                    elif "25G" in brkout_mode:
+                        port_speed = "25000"
+                    elif "50G" in brkout_mode:
+                        port_speed = "50000"
+                    elif "100G" in brkout_mode:
+                        port_speed = "100000"
+                    elif "200G" in brkout_mode:
+                        port_speed = "200000"
+
+                # Calculate individual lane for this breakout port
+                port_lanes = _calculate_breakout_port_lane(
+                    port_name, master_port, port_config
                 )
-            elif master_port in breakout_info["breakout_cfgs"]:
-                # Fallback to extracting speed from breakout mode
-                brkout_mode = breakout_info["breakout_cfgs"][master_port]["brkout_mode"]
-                if "10G" in brkout_mode:
-                    port_speed = "10000"
-                elif "25G" in brkout_mode:
-                    port_speed = "25000"
-                elif "50G" in brkout_mode:
-                    port_speed = "50000"
-                elif "100G" in brkout_mode:
-                    port_speed = "100000"
-                elif "200G" in brkout_mode:
-                    port_speed = "200000"
-
-            # Calculate individual lane for this breakout port
-            port_lanes = _calculate_breakout_port_lane(
-                port_name, master_port, port_config
-            )
 
         # Generate correct alias based on port name and speed
         interface_speed = int(port_speed) if port_speed else None
@@ -817,32 +833,47 @@ def _add_missing_breakout_ports(
     for port_name in breakout_info["breakout_ports"]:
         if port_name not in config["PORT"]:
             # Get the master port to determine configuration
-            master_port = breakout_info["breakout_ports"][port_name]["master"]
+            bp = breakout_info["breakout_ports"][port_name]
+            master_port = bp["master"]
 
-            # Override with individual breakout port speed from NetBox if available
-            # Note: netbox_interfaces speeds are already normalized to Mbps
-            if port_name in netbox_interfaces and netbox_interfaces[port_name]["speed"]:
-                port_speed = str(int(netbox_interfaces[port_name]["speed"]))
-                logger.debug(
-                    f"Using NetBox speed {port_speed} Mbps for missing breakout port {port_name}"
-                )
-            elif master_port in breakout_info["breakout_cfgs"]:
-                # Fallback to extracting speed from breakout mode
-                brkout_mode = breakout_info["breakout_cfgs"][master_port]["brkout_mode"]
-                if "10G" in brkout_mode:
-                    port_speed = "10000"
-                elif "25G" in brkout_mode:
-                    port_speed = "25000"
-                elif "50G" in brkout_mode:
-                    port_speed = "50000"
-                elif "100G" in brkout_mode:
-                    port_speed = "100000"
-                elif "200G" in brkout_mode:
-                    port_speed = "200000"
+            if bp.get("declared"):
+                port_speed = str(bp["speed"])
+                port_lanes = bp["lanes"]
+            else:
+                # Override with individual breakout port speed from NetBox if available
+                # Note: netbox_interfaces speeds are already normalized to Mbps
+                if (
+                    port_name in netbox_interfaces
+                    and netbox_interfaces[port_name]["speed"]
+                ):
+                    port_speed = str(int(netbox_interfaces[port_name]["speed"]))
+                    logger.debug(
+                        f"Using NetBox speed {port_speed} Mbps for missing breakout port {port_name}"
+                    )
+                elif master_port in breakout_info["breakout_cfgs"]:
+                    # Fallback to extracting speed from breakout mode
+                    brkout_mode = breakout_info["breakout_cfgs"][master_port][
+                        "brkout_mode"
+                    ]
+                    if "10G" in brkout_mode:
+                        port_speed = "10000"
+                    elif "25G" in brkout_mode:
+                        port_speed = "25000"
+                    elif "50G" in brkout_mode:
+                        port_speed = "50000"
+                    elif "100G" in brkout_mode:
+                        port_speed = "100000"
+                    elif "200G" in brkout_mode:
+                        port_speed = "200000"
+                    else:
+                        port_speed = "25000"  # Default fallback
                 else:
                     port_speed = "25000"  # Default fallback
-            else:
-                port_speed = "25000"  # Default fallback
+
+                # Calculate individual lane for this breakout port
+                port_lanes = _calculate_breakout_port_lane(
+                    port_name, master_port, port_config
+                )
 
             # Set admin_status based on connection or port channel membership
             admin_status = (
@@ -865,11 +896,6 @@ def _add_missing_breakout_ports(
             if master_port in port_config:
                 port_index = port_config[master_port]["index"]
 
-            # Calculate individual lane for this breakout port
-            port_lanes = _calculate_breakout_port_lane(
-                port_name, master_port, port_config
-            )
-
             port_data = {
                 "admin_status": admin_status,
                 "alias": correct_alias,
@@ -2393,14 +2419,14 @@ def _add_snmp_configuration(config, device, oob_ip):
     in the config_context of the device.
     """
 
-    # Create vault instance for Custom Field decryption
-    vault = get_vault()
-
-    # Decrypt secrets Custom Field
+    # Decrypt the secrets Custom Field. The vault needs the key file and
+    # Redis, and get_vault() logs ERRORs where those are absent, so it is
+    # only built when there are secrets to decrypt.
     node_secrets = device.custom_fields.get("secrets", {})
     if node_secrets is None:
         node_secrets = {}
-    deep_decrypt(node_secrets, vault)
+    if node_secrets:
+        deep_decrypt(node_secrets, get_vault())
 
     # Configure SNMP location and contact
     location = device.config_context.get("_segment_snmp_server_location", "Data Center")

osism/tasks/conductor/sonic/constants.py

@@ -2,6 +2,8 @@
 
 """Constants and mappings for SONiC configuration."""
 
+from osism import settings
+
 # Tag to add AF L2VPN EVPN to BGP neighbor
 BGP_AF_L2VPN_EVPN_TAG = "bgp-af-l2vpn-evpn"
 
@@ -87,7 +89,7 @@
 }
 
 # Path to SONiC port configuration files
-PORT_CONFIG_PATH = "/etc/sonic/port_config"
+PORT_CONFIG_PATH = settings.SONIC_PORT_CONFIG_PATH
 
 # List of supported vendors
 SUPPORTED_VENDORS = [