From 1145878f9efe9cc60853dd5ee88b6445562f44f6 Mon Sep 17 00:00:00 2001
From: Ryan Zezeski <ryan@oxide.computer>
Date: Tue, 9 Nov 2021 16:54:26 -0700
Subject: [PATCH 1/7] WIP conversation starter for OPTE + Sled Agent

I have not tested this nor even tried to compile it. I just wrote some
code on the fly to serve as a straw man for how I think Sled Agent +
OPTE interaction might look for the next demo milestone. In this
commit I only cover the registration/unregistration of the OPTE Port,
which is what overlays the illumos VNIC. Once again, this is messy and
stream of conciousness. Just wanna get the juices flowing in terms of
integration.
---
 sled-agent/src/instance.rs | 106 +++++++++++++++++++++++++++++++++++++
 1 file changed, 106 insertions(+)

diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index 5aed5299f9f..08a20561a02 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -163,10 +163,110 @@ impl Vnic {
         allocator: &IdAllocator,
         physical_dl: &PhysicalLink,
         mac: Option<MacAddr>,
+        ip: IpAddr,
         vlan: Option<VlanID>,
     ) -> Result<Self, Error> {
         let name = guest_vnic_name(allocator.next());
         Dladm::create_vnic(physical_dl, &name, mac, vlan)?;
+
+        // TODO (rpz): If vlan is Some(N), then OPTE is going to have
+        // a bad time...haven't been doing anything with VLANs. Easy
+        // enough to add, but need to know if this is something I need
+        // to implement before next demo.
+        //
+        // TODO (rpz): OPTE needs data in a reified state. I.e., it
+        // has no idea what a VPC UUID or VPC logical name means. It
+        // deals purely in concrete data like MAC addresses, IPs, and
+        // CIDRs. Until this data is wired up, we need to get grimey
+        // and assume the lab environment here for the demo.
+        let ip4 = match ip {
+            IpAddr::V4(v) => v,
+            _ => return Err("OPTE only supports IPv4 guest IPs at the moment"),
+        };
+
+        let public_mac = match mac {
+            Some(v) => {
+                let bytes = v.0.into_array();
+                bytes[0] = 0xA8;
+                bytes[1] = 0x40;
+                bytes[2] = 0x25;
+                EtherAddr::from(bytes)
+            },
+
+            // TODO (rpz): We could use allocator ID here, but
+            // honestly I just want this to go away. The only reason I
+            // added a "public mac" was so I could convince my home
+            // router that the `public_ip` is legit (it doesn't like
+            // having more than one IP with the same MAC address).
+            _ => EtherAddr::from([0xA8, 0x40, 0x25, 0x00, 0x00, 0x01])
+        };
+
+        // TODO (rpz): This is conjured from thin air and assumes
+        // the Oxide lab environment (see "Engineering Lab
+        // Network" in the meta repo). I decided to grab the
+        // 172.20.24.0/24 space. OPTE will pretend to be this IP
+        // and ARP for it.
+        let public_ip = Ipv4Addr::from((172, 20, 24, ip4.to_be_bytes()[0]));
+
+        // TODO (rpz): As the current "source NAT" is a bit of a lie
+        // (and I wrote it to work with my home network + terrible
+        // home router), it might make more sense to put the NAT
+        // config behind something like `Option<SrcNatConfig>`. This
+        // way we could specify `None` until we are ready to properly
+        // demo this behavior. With this in place, Sled Agent would
+        // need to provide only the following:
+        //
+        // * private_ip: `172.20.14.xxx`
+        // * gw_mac: `AA:00:04:00:FF:01`
+        // * gw_ip: `172.20.14.1`
+        //
+        // The private IP and gateway information are needed so that
+        // OPTE can proxy ARP in both directions. That is, instead of
+        // letting the guest VM ARP on the native IPv4 network, we
+        // make it look more like what the production Oxide Network
+        // will look like and only allow OPTE to generate ARP replies.
+        // This works because in the Oxide VPC Network the guest's IPs
+        // are always on a /32 or /128, which means all communication
+        // is "off link" and must hit the gateway. In the lab
+        // environment this is literally the actual IPv4 gateway, in
+        // the Oxide VPC Network this would be address XXX.XXX.XXX.1
+        // of the VPC subnet. In any event, the point is that we can
+        // get by with a lot less information by pushing SNAT to the
+        // side for now.
+        let ipcfg = IpConfig {
+            // TODO (rpz): For each interface a guest could have an
+            // IPv4 address, an IPv6 address, or both. Right now Sled
+            // Agent only allows specifying one or the other. And
+            // currently OPTE only allows IPv4 guest addresses.
+            //
+            // NOTE: OPTE has it's own Ipv4Addr, thus the into().
+            private_ip: ip4.into(),
+
+            public_mac,
+
+            public_ip,
+
+            // These ports are used to Source NAT flows into the
+            // public_ip.
+            port_start: 4096,
+            port_end: 8192,
+
+            // Once again, this is based on the Oxide lab environment.
+            vpc_sub4: opteadm::VpcSubnet4::new(VpcIpv4Cidr::new(ip4, 24)),
+            gw_mac: opteadm::EtherAddr::from(
+                [0xAA, 0x00, 0x04, 0x00, 0xFF, 0x01]
+            ),
+            gw_ip: "172.20.14.1".parse().unwrap(),
+        };
+
+        let req = RegisterPortReq {
+            link_name: name,
+            ip_cfg,
+        };
+
+        let hdl = opteadm::OpteAdm::open()?;
+        hdl.register_port(&req)?;
+
         Ok(Vnic { name, deleted: false })
     }
 
@@ -188,6 +288,11 @@ impl Vnic {
             Ok(())
         } else {
             self.deleted = true;
+            // TODO (rpz): The `hdl` is something we could create once
+            // and stash in `self`, but it's also fine to open a new
+            // one for each operation.
+            let hdl = opteadm::OpteAdm::open()?;
+            hdl.unregister_port(&self.name)?;
             Dladm::delete_vnic(&self.name)
         }
     }
@@ -443,6 +548,7 @@ impl Instance {
                     &inner.nic_id_allocator,
                     &physical_dl,
                     Some(nic.mac),
+                    nic.ip,
                     inner.vlan,
                 )
             })

From 77885a147f7acf31cbdee9d219206ee9f819d67e Mon Sep 17 00:00:00 2001
From: Ryan Zezeski <ryan@oxide.computer>
Date: Tue, 23 Nov 2021 17:15:34 -0700
Subject: [PATCH 2/7] WIP make SNAT optional, hard-code NIC

---
 nexus/src/sagas.rs         | 16 +++++++++++-
 sled-agent/src/instance.rs | 51 +++-----------------------------------
 2 files changed, 18 insertions(+), 49 deletions(-)

diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs
index a29deb9ddb8..3536335a749 100644
--- a/nexus/src/sagas.rs
+++ b/nexus/src/sagas.rs
@@ -153,11 +153,25 @@ async fn sic_create_instance_record(
         .await
         .map_err(ActionError::action_failed)?;
 
+    let rpz_nic = NetworkInterface {
+        identity: IdentityMetadata {
+            id: Uuid::new_v4(),
+            name: "rpz_nic".parse().unwrap(),
+            description: "test nic",
+            time_created: Utc::now(),
+            time_modified: Utc::now(),
+        },
+        vpc_id: Uuid::new_v4(),
+        subnet_id: Uuid::new_v4(),
+        mac: MacAddr::V6(MacAddr6::from([0xA8, 0x40, 0x25, 0x00, 0x00, 0xD5]),
+        ip: IpAddr::V4(Ipv4Addr::new(10, 0, 0, 213)),
+    };
+
     // TODO: Populate this with an appropriate NIC.
     // See also: instance_set_runtime in nexus.rs for a similar construction.
     Ok(InstanceHardware {
         runtime: instance.runtime().clone().into(),
-        nics: vec![],
+        nics: vec![rpz_nic],
     })
 }
 
diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index 08a20561a02..50ca7110f00 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -184,37 +184,8 @@ impl Vnic {
             _ => return Err("OPTE only supports IPv4 guest IPs at the moment"),
         };
 
-        let public_mac = match mac {
-            Some(v) => {
-                let bytes = v.0.into_array();
-                bytes[0] = 0xA8;
-                bytes[1] = 0x40;
-                bytes[2] = 0x25;
-                EtherAddr::from(bytes)
-            },
-
-            // TODO (rpz): We could use allocator ID here, but
-            // honestly I just want this to go away. The only reason I
-            // added a "public mac" was so I could convince my home
-            // router that the `public_ip` is legit (it doesn't like
-            // having more than one IP with the same MAC address).
-            _ => EtherAddr::from([0xA8, 0x40, 0x25, 0x00, 0x00, 0x01])
-        };
-
-        // TODO (rpz): This is conjured from thin air and assumes
-        // the Oxide lab environment (see "Engineering Lab
-        // Network" in the meta repo). I decided to grab the
-        // 172.20.24.0/24 space. OPTE will pretend to be this IP
-        // and ARP for it.
-        let public_ip = Ipv4Addr::from((172, 20, 24, ip4.to_be_bytes()[0]));
-
-        // TODO (rpz): As the current "source NAT" is a bit of a lie
-        // (and I wrote it to work with my home network + terrible
-        // home router), it might make more sense to put the NAT
-        // config behind something like `Option<SrcNatConfig>`. This
-        // way we could specify `None` until we are ready to properly
-        // demo this behavior. With this in place, Sled Agent would
-        // need to provide only the following:
+        // TODO (rpz): For now we assume the lab environment for the
+        // sake of the demo.
         //
         // * private_ip: `172.20.14.xxx`
         // * gw_mac: `AA:00:04:00:FF:01`
@@ -234,29 +205,13 @@ impl Vnic {
         // get by with a lot less information by pushing SNAT to the
         // side for now.
         let ipcfg = IpConfig {
-            // TODO (rpz): For each interface a guest could have an
-            // IPv4 address, an IPv6 address, or both. Right now Sled
-            // Agent only allows specifying one or the other. And
-            // currently OPTE only allows IPv4 guest addresses.
-            //
             // NOTE: OPTE has it's own Ipv4Addr, thus the into().
             private_ip: ip4.into(),
-
-            public_mac,
-
-            public_ip,
-
-            // These ports are used to Source NAT flows into the
-            // public_ip.
-            port_start: 4096,
-            port_end: 8192,
-
-            // Once again, this is based on the Oxide lab environment.
-            vpc_sub4: opteadm::VpcSubnet4::new(VpcIpv4Cidr::new(ip4, 24)),
             gw_mac: opteadm::EtherAddr::from(
                 [0xAA, 0x00, 0x04, 0x00, 0xFF, 0x01]
             ),
             gw_ip: "172.20.14.1".parse().unwrap(),
+	    snat: None,
         };
 
         let req = RegisterPortReq {

From 862355219da97eb913f6a54f305beab35b781e4a Mon Sep 17 00:00:00 2001
From: Ryan Zezeski <ryan@oxide.computer>
Date: Tue, 23 Nov 2021 17:46:44 -0700
Subject: [PATCH 3/7] WIP add opteadm dep

---
 sled-agent/Cargo.toml      | 1 +
 sled-agent/src/instance.rs | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml
index 5a33b7f09b6..f467fd4f8e4 100644
--- a/sled-agent/Cargo.toml
+++ b/sled-agent/Cargo.toml
@@ -13,6 +13,7 @@ futures = "0.3.18"
 ipnetwork = "0.18"
 nexus-client = { path = "../nexus-client" }
 omicron-common = { path = "../common" }
+opteadm = { path = "../../opte/opteadm" }
 percent-encoding = "2.1.0"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
 propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "b56e473" }
diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index 8ff318a98c8..b4c0d32ac45 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -20,7 +20,7 @@ use omicron_common::api::internal::sled_agent::InstanceRuntimeStateRequested;
 use omicron_common::backoff;
 use propolis_client::Client as PropolisClient;
 use slog::Logger;
-use std::net::SocketAddr;
+use std::net::{IpAddr, SocketAddr};
 use std::sync::Arc;
 use tokio::task::JoinHandle;
 use uuid::Uuid;
@@ -204,7 +204,7 @@ impl Vnic {
         // of the VPC subnet. In any event, the point is that we can
         // get by with a lot less information by pushing SNAT to the
         // side for now.
-        let ipcfg = IpConfig {
+        let ip_cfg = IpConfig {
             // NOTE: OPTE has it's own Ipv4Addr, thus the into().
             private_ip: ip4.into(),
             gw_mac: opteadm::EtherAddr::from(

From 53877bd71e213ad55d5cc5549256953da5e0f79f Mon Sep 17 00:00:00 2001
From: Ryan Zezeski <ryan@oxide.computer>
Date: Tue, 30 Nov 2021 07:36:56 -0700
Subject: [PATCH 4/7] WIP some fixes

---
 common/Cargo.toml                |  1 +
 common/src/api/external/error.rs |  6 ++++++
 nexus/src/sagas.rs               | 11 ++++++++---
 sled-agent/Cargo.toml            |  1 +
 sled-agent/src/instance.rs       | 17 ++++++++++++-----
 5 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/common/Cargo.toml b/common/Cargo.toml
index 49f5bfb403a..50150bd258a 100644
--- a/common/Cargo.toml
+++ b/common/Cargo.toml
@@ -10,6 +10,7 @@ futures = "0.3.18"
 http = "0.2.5"
 hyper = "0.14"
 ipnetwork = "0.18"
+opteadm = { path = "../../opte/opteadm" }
 propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "b56e473" }
 rayon = "1.5"
 reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] }
diff --git a/common/src/api/external/error.rs b/common/src/api/external/error.rs
index 4d8d1470f7a..e691d1b78e2 100644
--- a/common/src/api/external/error.rs
+++ b/common/src/api/external/error.rs
@@ -262,6 +262,12 @@ impl From<anyhow::Error> for crate::api::external::Error {
     }
 }
 
+impl From<opteadm::Error> for Error {
+    fn from(err: opteadm::Error) -> Self {
+        Error::InternalError { internal_message: format!("{}", err) }
+    }
+}
+
 /**
  * Like [`assert!`], except that instead of panicking, this function returns an
  * `Err(Error::InternalError)` with an appropriate message if the given
diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs
index 0e48fddda8b..9f6a0b02c13 100644
--- a/nexus/src/sagas.rs
+++ b/nexus/src/sagas.rs
@@ -13,9 +13,12 @@ use crate::db;
 use crate::saga_interface::SagaContext;
 use chrono::Utc;
 use lazy_static::lazy_static;
+use omicron_common::api::external;
 use omicron_common::api::external::Generation;
+use omicron_common::api::external::IdentityMetadata;
 use omicron_common::api::external::InstanceCreateParams;
 use omicron_common::api::external::InstanceState;
+use omicron_common::api::external::NetworkInterface;
 use omicron_common::api::internal::nexus::InstanceRuntimeState;
 use omicron_common::api::internal::sled_agent::InstanceHardware;
 use serde::Deserialize;
@@ -158,14 +161,16 @@ async fn sic_create_instance_record(
         identity: IdentityMetadata {
             id: Uuid::new_v4(),
             name: "rpz_nic".parse().unwrap(),
-            description: "test nic",
+            description: "test nic".to_string(),
             time_created: Utc::now(),
             time_modified: Utc::now(),
         },
         vpc_id: Uuid::new_v4(),
         subnet_id: Uuid::new_v4(),
-        mac: MacAddr::V6(MacAddr6::from([0xA8, 0x40, 0x25, 0x00, 0x00, 0xD5]),
-        ip: IpAddr::V4(Ipv4Addr::new(10, 0, 0, 213)),
+        mac: external::MacAddr(
+            macaddr::MacAddr6::from([0xA8, 0x40, 0x25, 0x00, 0x00, 0xD5])
+        ),
+        ip: std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 0, 0, 213)),
     };
 
     // TODO: Populate this with an appropriate NIC.
diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml
index f467fd4f8e4..98e301710de 100644
--- a/sled-agent/Cargo.toml
+++ b/sled-agent/Cargo.toml
@@ -13,6 +13,7 @@ futures = "0.3.18"
 ipnetwork = "0.18"
 nexus-client = { path = "../nexus-client" }
 omicron-common = { path = "../common" }
+opte-core = { path = "../../opte/opte-core" }
 opteadm = { path = "../../opte/opteadm" }
 percent-encoding = "2.1.0"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index b4c0d32ac45..2d4fff2dfff 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -181,7 +181,14 @@ impl Vnic {
         // and assume the lab environment here for the demo.
         let ip4 = match ip {
             IpAddr::V4(v) => v,
-            _ => return Err("OPTE only supports IPv4 guest IPs at the moment"),
+
+            _ => {
+                return Err(Error::InternalError {
+                    internal_message: format!(
+                        "OPTE only supports IPv4 guests at the moment",
+                    ),
+                });
+            }
         };
 
         // TODO (rpz): For now we assume the lab environment for the
@@ -204,18 +211,18 @@ impl Vnic {
         // of the VPC subnet. In any event, the point is that we can
         // get by with a lot less information by pushing SNAT to the
         // side for now.
-        let ip_cfg = IpConfig {
+        let ip_cfg = opte_core::ioctl::IpConfig {
             // NOTE: OPTE has it's own Ipv4Addr, thus the into().
             private_ip: ip4.into(),
-            gw_mac: opteadm::EtherAddr::from(
+            gw_mac: opte_core::ether::EtherAddr::from(
                 [0xAA, 0x00, 0x04, 0x00, 0xFF, 0x01]
             ),
             gw_ip: "172.20.14.1".parse().unwrap(),
 	    snat: None,
         };
 
-        let req = RegisterPortReq {
-            link_name: name,
+        let req = opte_core::ioctl::RegisterPortReq {
+            link_name: name.clone(),
             ip_cfg,
         };
 

From e8b1296d23ebbdf5d4b1197832ba45271b0fddad Mon Sep 17 00:00:00 2001
From: Ryan Zezeski <ryan@oxide.computer>
Date: Thu, 2 Dec 2021 08:31:27 -0700
Subject: [PATCH 5/7] WIP stash

---
 common/src/api/external/mod.rs | 15 ++++++++++++---
 nexus/src/nexus.rs             | 21 ++++++++++++++++++++-
 nexus/src/sagas.rs             |  4 ++--
 sled-agent/src/instance.rs     | 13 +++++++++++--
 4 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs
index 5984551732c..caa3b739986 100644
--- a/common/src/api/external/mod.rs
+++ b/common/src/api/external/mod.rs
@@ -1921,14 +1921,23 @@ impl JsonSchema for L4PortRange {
 /// hardware devices on a network.
 // NOTE: We're using the `macaddr` crate for the internal representation. But as with the `ipnet`,
 // this crate does not implement `JsonSchema`.
-#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)]
+#[derive(Clone, Copy, Debug, DeserializeFromStr, PartialEq, SerializeDisplay)]
 pub struct MacAddr(pub macaddr::MacAddr6);
 
+impl FromStr for MacAddr {
+     type Err = macaddr::ParseError;
+
+     fn from_str(s: &str) -> Result<Self, Self::Err> {
+         s.parse().map(|addr| MacAddr(addr))
+     }
+}
+
 impl TryFrom<String> for MacAddr {
-    type Error = macaddr::ParseError;
+    type Error = <Self as FromStr>::Err;
 
     fn try_from(s: String) -> Result<Self, Self::Error> {
-        s.parse().map(|addr| MacAddr(addr))
+        // s.parse().map(|addr| MacAddr(addr))
+        MacAddr::from_str(s.as_ref())
     }
 }
 
diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs
index b3db597f058..e2119e8c30c 100644
--- a/nexus/src/nexus.rs
+++ b/nexus/src/nexus.rs
@@ -14,6 +14,7 @@ use crate::saga_interface::SagaContext;
 use crate::sagas;
 use anyhow::Context;
 use async_trait::async_trait;
+use chrono::Utc;
 use futures::future::ready;
 use futures::StreamExt;
 use hex;
@@ -1068,6 +1069,24 @@ impl Nexus {
         let runtime: nexus::InstanceRuntimeState =
             instance.runtime().clone().into();
 
+        let rpz_nic = sled_agent_client::types::NetworkInterface {
+            identity: sled_agent_client::types::IdentityMetadata {
+                id: Uuid::new_v4(),
+                name: sled_agent_client::types::Name("rpz-nic".to_string()),
+                description: "test nic".to_string(),
+                time_created: Utc::now(),
+                time_modified: Utc::now(),
+            },
+            vpc_id: Uuid::new_v4(),
+            subnet_id: Uuid::new_v4(),
+            mac: sled_agent_client::types::MacAddr(
+                format!("02:08:20:BE:A0:F2")
+            ),
+            ip: std::net::IpAddr::V4(
+                std::net::Ipv4Addr::new(10, 0, 0, 213)
+            ).to_string(),
+        };
+
         // TODO: Populate this with an appropriate NIC.
         // See also: sic_create_instance_record in sagas.rs for a similar
         // construction.
@@ -1075,7 +1094,7 @@ impl Nexus {
             runtime: sled_agent_client::types::InstanceRuntimeState::from(
                 runtime,
             ),
-            nics: vec![],
+            nics: vec![rpz_nic],
         };
 
         let new_runtime = sa
diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs
index 9f6a0b02c13..478d1373766 100644
--- a/nexus/src/sagas.rs
+++ b/nexus/src/sagas.rs
@@ -160,7 +160,7 @@ async fn sic_create_instance_record(
     let rpz_nic = NetworkInterface {
         identity: IdentityMetadata {
             id: Uuid::new_v4(),
-            name: "rpz_nic".parse().unwrap(),
+            name: "rpz-nic".parse().unwrap(),
             description: "test nic".to_string(),
             time_created: Utc::now(),
             time_modified: Utc::now(),
@@ -168,7 +168,7 @@ async fn sic_create_instance_record(
         vpc_id: Uuid::new_v4(),
         subnet_id: Uuid::new_v4(),
         mac: external::MacAddr(
-            macaddr::MacAddr6::from([0xA8, 0x40, 0x25, 0x00, 0x00, 0xD5])
+            macaddr::MacAddr6::from([0x02, 0x08, 0x20, 0xBE, 0xA0, 0xF2])
         ),
         ip: std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 0, 0, 213)),
     };
diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index 2d4fff2dfff..8f2339a981e 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -214,10 +214,19 @@ impl Vnic {
         let ip_cfg = opte_core::ioctl::IpConfig {
             // NOTE: OPTE has it's own Ipv4Addr, thus the into().
             private_ip: ip4.into(),
+            // This is lab config.
+            //
+            // gw_mac: opte_core::ether::EtherAddr::from(
+            //     [0xAA, 0x00, 0x04, 0x00, 0xFF, 0x01]
+            // ),
+            // gw_ip: "172.20.14.1".parse().unwrap(),
+
+            // This is home config.
             gw_mac: opte_core::ether::EtherAddr::from(
-                [0xAA, 0x00, 0x04, 0x00, 0xFF, 0x01]
+                [0x78, 0x23, 0xAE, 0x5D, 0x4F, 0x0D]
             ),
-            gw_ip: "172.20.14.1".parse().unwrap(),
+            gw_ip: "10.0.0.1".parse().unwrap(),
+
 	    snat: None,
         };
 

From 009827b3d886e627c65881f98444a0e718b1ea0e Mon Sep 17 00:00:00 2001
From: Ryan Zezeski <ryan@oxide.computer>
Date: Mon, 6 Dec 2021 21:25:46 -0700
Subject: [PATCH 6/7] WIP update for latest opte

---
 sled-agent/src/instance.rs | 52 +++-----------------------------------
 1 file changed, 3 insertions(+), 49 deletions(-)

diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index 8f2339a981e..0cd675dca7f 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -169,16 +169,6 @@ impl Vnic {
         let name = guest_vnic_name(allocator.next());
         Dladm::create_vnic(physical_dl, &name, mac, vlan)?;
 
-        // TODO (rpz): If vlan is Some(N), then OPTE is going to have
-        // a bad time...haven't been doing anything with VLANs. Easy
-        // enough to add, but need to know if this is something I need
-        // to implement before next demo.
-        //
-        // TODO (rpz): OPTE needs data in a reified state. I.e., it
-        // has no idea what a VPC UUID or VPC logical name means. It
-        // deals purely in concrete data like MAC addresses, IPs, and
-        // CIDRs. Until this data is wired up, we need to get grimey
-        // and assume the lab environment here for the demo.
         let ip4 = match ip {
             IpAddr::V4(v) => v,
 
@@ -191,52 +181,19 @@ impl Vnic {
             }
         };
 
-        // TODO (rpz): For now we assume the lab environment for the
-        // sake of the demo.
-        //
-        // * private_ip: `172.20.14.xxx`
-        // * gw_mac: `AA:00:04:00:FF:01`
-        // * gw_ip: `172.20.14.1`
-        //
-        // The private IP and gateway information are needed so that
-        // OPTE can proxy ARP in both directions. That is, instead of
-        // letting the guest VM ARP on the native IPv4 network, we
-        // make it look more like what the production Oxide Network
-        // will look like and only allow OPTE to generate ARP replies.
-        // This works because in the Oxide VPC Network the guest's IPs
-        // are always on a /32 or /128, which means all communication
-        // is "off link" and must hit the gateway. In the lab
-        // environment this is literally the actual IPv4 gateway, in
-        // the Oxide VPC Network this would be address XXX.XXX.XXX.1
-        // of the VPC subnet. In any event, the point is that we can
-        // get by with a lot less information by pushing SNAT to the
-        // side for now.
         let ip_cfg = opte_core::ioctl::IpConfig {
             // NOTE: OPTE has it's own Ipv4Addr, thus the into().
             private_ip: ip4.into(),
-            // This is lab config.
-            //
-            // gw_mac: opte_core::ether::EtherAddr::from(
-            //     [0xAA, 0x00, 0x04, 0x00, 0xFF, 0x01]
-            // ),
-            // gw_ip: "172.20.14.1".parse().unwrap(),
-
-            // This is home config.
-            gw_mac: opte_core::ether::EtherAddr::from(
-                [0x78, 0x23, 0xAE, 0x5D, 0x4F, 0x0D]
-            ),
-            gw_ip: "10.0.0.1".parse().unwrap(),
-
 	    snat: None,
         };
 
-        let req = opte_core::ioctl::RegisterPortReq {
+        let req = opte_core::ioctl::AddPortReq {
             link_name: name.clone(),
             ip_cfg,
         };
 
         let hdl = opteadm::OpteAdm::open()?;
-        hdl.register_port(&req)?;
+        hdl.add_port(&req)?;
 
         Ok(Vnic { name, deleted: false })
     }
@@ -259,11 +216,8 @@ impl Vnic {
             Ok(())
         } else {
             self.deleted = true;
-            // TODO (rpz): The `hdl` is something we could create once
-            // and stash in `self`, but it's also fine to open a new
-            // one for each operation.
             let hdl = opteadm::OpteAdm::open()?;
-            hdl.unregister_port(&self.name)?;
+            hdl.delete_port(&self.name)?;
             Dladm::delete_vnic(&self.name)
         }
     }

From 2a748cb90229a3be3e9305cd7cee237e3ad412c9 Mon Sep 17 00:00:00 2001
From: Ryan Zezeski <ryan@oxide.computer>
Date: Tue, 7 Dec 2021 14:21:42 -0700
Subject: [PATCH 7/7] WIP some fixes after latest merge

---
 sled-agent/src/illumos/dladm.rs | 5 +++++
 sled-agent/src/instance.rs      | 2 +-
 sled-agent/src/vnic.rs          | 7 ++-----
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/sled-agent/src/illumos/dladm.rs b/sled-agent/src/illumos/dladm.rs
index 62db17600c9..f755e956aea 100644
--- a/sled-agent/src/illumos/dladm.rs
+++ b/sled-agent/src/illumos/dladm.rs
@@ -24,6 +24,11 @@ pub enum Error {
 
     #[error("Failed to parse output: {0}")]
     Parse(#[from] std::string::FromUtf8Error),
+
+    // TODO: This isn't the right place to put this, but I just want
+    // to get integration going.
+    #[error("OPTE error: {0}")]
+    OpteErr(#[from] opteadm::Error),
 }
 
 /// The name of a physical datalink.
diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index 4df48483792..ff3d70fa75d 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -20,7 +20,7 @@ use omicron_common::api::internal::sled_agent::InstanceRuntimeStateRequested;
 use omicron_common::backoff;
 use propolis_client::Client as PropolisClient;
 use slog::Logger;
-use std::net::{IpAddr, SocketAddr};
+use std::net::SocketAddr;
 use std::sync::Arc;
 use tokio::task::JoinHandle;
 use uuid::Uuid;
diff --git a/sled-agent/src/vnic.rs b/sled-agent/src/vnic.rs
index 7c1b2215f0c..1c2b6e4a364 100644
--- a/sled-agent/src/vnic.rs
+++ b/sled-agent/src/vnic.rs
@@ -9,6 +9,7 @@ use crate::illumos::dladm::{
     PhysicalLink, VNIC_PREFIX_CONTROL, VNIC_PREFIX_GUEST,
 };
 use omicron_common::api::external::MacAddr;
+use std::net::IpAddr;
 use std::sync::{
     atomic::{AtomicU64, Ordering},
     Arc,
@@ -83,11 +84,7 @@ impl Vnic {
             IpAddr::V4(v) => v,
 
             _ => {
-                return Err(Error::InternalError {
-                    internal_message: format!(
-                        "OPTE only supports IPv4 guests at the moment",
-                    ),
-                });
+                todo!("OPTE supports IPv4 guests only at the moment");
             }
         };
         let ip_cfg = opte_core::ioctl::IpConfig {