diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql index 5a72569ee99..74978063159 100644 --- a/common/src/sql/dbinit.sql +++ b/common/src/sql/dbinit.sql @@ -959,6 +959,14 @@ CREATE TABLE omicron.public.role_assignment_builtin ( PRIMARY KEY(user_builtin_id, resource_type, resource_id, role_name) ); +/* + * Store addresses allocated for internal services (like propolis zones) + */ +CREATE TABLE omicron.public.static_v6_address ( + address INET PRIMARY KEY NOT NULL, + associated_id UUID NOT NULL +); + /*******************************************************************/ /* diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index e26d30efcca..c4a42ba0cdb 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -55,6 +55,7 @@ use omicron_common::api::external::{ use omicron_common::api::internal::nexus::UpdateArtifact; use omicron_common::bail_unless; use std::convert::{TryFrom, TryInto}; +use std::net::IpAddr; use std::sync::Arc; use uuid::Uuid; @@ -67,9 +68,10 @@ use crate::db::{ Name, NetworkInterface, Organization, OrganizationUpdate, OximeterInfo, ProducerEndpoint, Project, ProjectUpdate, Region, RoleAssignmentBuiltin, RoleBuiltin, RouterRoute, RouterRouteUpdate, - Silo, SiloUser, Sled, UpdateArtifactKind, UpdateAvailableArtifact, - UserBuiltin, Volume, Vpc, VpcFirewallRule, VpcRouter, VpcRouterUpdate, - VpcSubnet, VpcSubnetUpdate, VpcUpdate, Zpool, + Silo, SiloUser, Sled, StaticV6Address, UpdateArtifactKind, + UpdateAvailableArtifact, UserBuiltin, Volume, Vpc, VpcFirewallRule, + VpcRouter, VpcRouterUpdate, VpcSubnet, VpcSubnetUpdate, VpcUpdate, + Zpool, }, pagination::paginated, pagination::paginated_multicolumn, @@ -1312,6 +1314,16 @@ impl DataStore { let failed = DbInstanceState::new(ApiInstanceState::Failed); let instance_id = authz_instance.id(); + + // XXX delete allocated ip + // XXX what if the address isn't in the table? + // XXX do in transaction (saga?) + // + //use db::schema::static_v6_address::dsl as address_dsl + //diesel::delete(address_dsl::static_v6_address) + // .filter(address_dsl::associated_id.eq(instance_id)) + // .execute(conn)?; + let result = diesel::update(dsl::instance) .filter(dsl::time_deleted.is_null()) .filter(dsl::id.eq(instance_id)) @@ -1326,6 +1338,7 @@ impl DataStore { ErrorHandler::NotFoundByResource(authz_instance), ) })?; + match result.status { UpdateStatus::Updated => Ok(()), UpdateStatus::NotUpdatedButExists => { @@ -3714,6 +3727,93 @@ impl DataStore { Ok(()) } + + pub async fn allocate_static_v6_address( + &self, + associated_id: Uuid, + ) -> Result { + use db::schema::static_v6_address::dsl; + + self.pool() + .transaction(move |conn| { + // Grab the next sequential address + // XXX should this use push_next_available_ip_subquery? + // XXX refactor push_next_available_ip_subquery? + // XXX this is a full table scan + let query = vec![ + "select".to_string(), + format!("'{}'::UUID", associated_id), + "as associated_id, static_v6_address.address + 1 as address".to_string(), + "from static_v6_address where not exists".to_string(), + "(select 1 from static_v6_address tmp where tmp.address = static_v6_address.address + 1)".to_string(), + "limit 1".to_string(), + ]; + let new_address: Vec = diesel::sql_query(query.join(" ")).load(conn)?; + + let new_address = if new_address.len() != 1 { + // This means there are no addresses in the table! + // XXX normally this base address should come from something + // else, hard code here. + StaticV6Address { + associated_id, + address: "fd00:1234::101".parse().unwrap(), + } + } else { + new_address[0].clone() + }; + + let new_address = diesel::insert_into(dsl::static_v6_address) + .values(new_address) + .returning(StaticV6Address::as_returning()) + .get_result(conn)?; + + return Ok(new_address.address.ip()); + }) + .await + .map_err(|e: async_bb8_diesel::PoolError| { + Error::internal_error(&format!("Transaction error: {}", e)) + }) + } + + pub async fn static_v6_address_fetch( + &self, + associated_id: Uuid, + ) -> LookupResult { + use db::schema::static_v6_address::dsl; + + dsl::static_v6_address + .filter(dsl::associated_id.eq(associated_id)) + .select(StaticV6Address::as_select()) + .first_async(self.pool()) + .await + .map(|x| x.address.ip()) + .map_err(|e| { + Error::internal_error(&format!( + "error fetching static v6 address for {:?}: {:?}", + associated_id, e + )) + }) + } + + pub async fn free_static_v6_address( + &self, + address: IpAddr, + ) -> UpdateResult<()> { + use db::schema::static_v6_address::dsl; + + // XXX what if the address isn't in the table? + diesel::delete(dsl::static_v6_address) + .filter(dsl::address.eq(ipnetwork::IpNetwork::from(address))) + .execute_async(self.pool()) + .await + .map(|_| ()) + .map_err(|e| { + Error::internal_error(&format!( + "error freeing static v6 address: {:?}", + e + )) + }) + } } /// Constructs a DataStore for use in test suites that has preloaded the @@ -3766,7 +3866,7 @@ mod test { }; use omicron_test_utils::dev; use std::collections::HashSet; - use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; use uuid::Uuid; @@ -4213,4 +4313,83 @@ mod test { let _ = db.cleanup().await; } + + // Test static v6 address allocation + #[tokio::test] + async fn test_static_v6_address_allocation() -> Result<(), Error> { + let logctx = dev::test_setup_log("test_static_v6_address_allocation"); + let mut db = test_setup_database(&logctx.log).await; + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = db::Pool::new(&cfg); + let datastore = DataStore::new(Arc::new(pool)); + + // db contents: [] + let address1 = + datastore.allocate_static_v6_address(Uuid::new_v4()).await?; + // db contents: [101] + assert_eq!(address1, "fd00:1234::101".parse::().unwrap()); + + // db contents: [101] + let address2 = + datastore.allocate_static_v6_address(Uuid::new_v4()).await?; + // db contents: [101, 102] + assert_eq!(address2, "fd00:1234::102".parse::().unwrap()); + + // db contents: [101, 102] + let address3 = + datastore.allocate_static_v6_address(Uuid::new_v4()).await?; + // db contents: [101, 102, 103] + assert_eq!(address3, "fd00:1234::103".parse::().unwrap()); + + // db contents: [101, 102, 103] + datastore.free_static_v6_address(address1).await?; + // db contents: [102, 103] + + // db contents: [102, 103] + let address4 = + datastore.allocate_static_v6_address(Uuid::new_v4()).await?; + // db contents: [102, 103, 104] + assert_eq!(address4, "fd00:1234::104".parse::().unwrap()); + + // db contents: [102, 103, 104] + datastore.free_static_v6_address(address4).await?; + // db contents: [102, 103] + + // db contents: [102, 103] + datastore.free_static_v6_address(address2).await?; + // db contents: [103] + + // db contents: [103] + let address5 = + datastore.allocate_static_v6_address(Uuid::new_v4()).await?; + // db contents: [103, 104] + assert_eq!(address5, "fd00:1234::104".parse::().unwrap()); + + // db contents: [103, 104] + let address6 = + datastore.allocate_static_v6_address(Uuid::new_v4()).await?; + // db contents: [103, 104, 105] + assert_eq!(address6, "fd00:1234::105".parse::().unwrap()); + + // db contents: [103, 104, 105] + let address7 = + datastore.allocate_static_v6_address(Uuid::new_v4()).await?; + // db contents: [103, 104, 105, 106] + assert_eq!(address7, "fd00:1234::106".parse::().unwrap()); + + // get associated address + let id = Uuid::new_v4(); + let address = datastore.allocate_static_v6_address(id.clone()).await?; + assert_eq!(address, datastore.static_v6_address_fetch(id).await?); + + // assert that getting the address of an unknown ID returns an error + assert!(datastore + .static_v6_address_fetch(Uuid::new_v4()) + .await + .is_err()); + + let _ = db.cleanup().await; + + Ok(()) + } } diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index 5f51ec218b1..55ab08ad5f3 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -10,8 +10,8 @@ use crate::db::schema::{ console_session, dataset, disk, instance, metric_producer, network_interface, organization, oximeter, project, rack, region, role_assignment_builtin, role_builtin, router_route, silo, silo_user, sled, - snapshot, update_available_artifact, user_builtin, volume, vpc, - vpc_firewall_rule, vpc_router, vpc_subnet, zpool, + snapshot, static_v6_address, update_available_artifact, user_builtin, + volume, vpc, vpc_firewall_rule, vpc_router, vpc_subnet, zpool, }; use crate::defaults; use crate::external_api::params; @@ -2460,6 +2460,21 @@ pub struct UpdateAvailableArtifact { pub target_length: i64, } +#[derive( + Queryable, + QueryableByName, + Insertable, + Clone, + Debug, + Selectable, + AsChangeset, +)] +#[table_name = "static_v6_address"] +pub struct StaticV6Address { + pub address: IpNetwork, + pub associated_id: Uuid, +} + #[cfg(test)] mod tests { use super::Uuid; diff --git a/nexus/src/db/schema.rs b/nexus/src/db/schema.rs index 4353f5e375f..bea5c1d0f1e 100644 --- a/nexus/src/db/schema.rs +++ b/nexus/src/db/schema.rs @@ -399,6 +399,13 @@ table! { } } +table! { + static_v6_address (address) { + address -> Inet, + associated_id -> Uuid, + } +} + allow_tables_to_appear_in_same_query!( dataset, disk, diff --git a/nexus/src/internal_api/params.rs b/nexus/src/internal_api/params.rs index 96f078b9c6d..a2fde8c7a67 100644 --- a/nexus/src/internal_api/params.rs +++ b/nexus/src/internal_api/params.rs @@ -7,7 +7,7 @@ use omicron_common::api::external::ByteCount; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fmt; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use std::str::FromStr; use uuid::Uuid; @@ -105,3 +105,15 @@ pub struct OximeterInfo { /// The address on which this oximeter instance listens for requests pub address: SocketAddr, } + +/// Response when allocating a static v6 address +#[derive(Debug, Clone, Copy, JsonSchema, Serialize, Deserialize)] +pub struct AllocateStaticV6AddressResponse { + pub address: IpAddr, +} + +/// Static Ipv6 address to free +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct FreeStaticV6AddressParams { + pub address: IpAddr, +} diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index c5b27852646..457cb76930f 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -70,7 +70,7 @@ use sled_agent_client::types::InstanceStateRequested; use sled_agent_client::Client as SledAgentClient; use slog::Logger; use std::convert::{TryFrom, TryInto}; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use std::num::NonZeroU32; use std::path::Path; use std::sync::Arc; @@ -1370,6 +1370,9 @@ impl Nexus { disks: disk_reqs, }; + let allocated_control_ip = + self.db_datastore.static_v6_address_fetch(db_instance.id()).await?; + let sa = self.instance_sled(&db_instance).await?; let new_runtime = sa @@ -1379,6 +1382,7 @@ impl Nexus { initial: instance_hardware, target: requested, migrate: None, + allocated_control_ip: allocated_control_ip.to_string(), }, ) .await @@ -3287,6 +3291,20 @@ impl Nexus { ) -> LookupResult { self.db_datastore.silo_user_fetch(silo_user_id).await } + + pub async fn allocate_static_v6_address( + &self, + associated_id: Uuid, + ) -> Result { + self.db_datastore.allocate_static_v6_address(associated_id).await + } + + pub async fn free_static_v6_address( + &self, + address: IpAddr, + ) -> Result<(), Error> { + self.db_datastore.free_static_v6_address(address).await + } } fn generate_session_token() -> String { diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 6cb886f9b4c..2cc30fdc4fb 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -43,6 +43,7 @@ use slog::warn; use slog::Logger; use std::collections::BTreeMap; use std::convert::{TryFrom, TryInto}; +use std::net::IpAddr; use std::sync::Arc; use steno::new_action_noop_undo; use steno::ActionContext; @@ -241,6 +242,15 @@ pub fn saga_instance_create() -> SagaTemplate { ); } + template_builder.append( + "allocated_control_ip", + "AllocateIpv6ForPropolisControl", + ActionFunc::new_action( + sic_allocate_v6_address, + sic_allocate_v6_address_undo, + ), + ); + template_builder.append( "instance_ensure", "InstanceEnsure", @@ -752,6 +762,50 @@ async fn sic_delete_instance_record( Ok(()) } +// XXX can't reuse this function because ActionContext impl changes? + +async fn sic_allocate_v6_address( + sagactx: ActionContext, +) -> Result { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let instance_id = sagactx.lookup::("instance_id")?; + Ok(nexus + .allocate_static_v6_address(instance_id) + .await + .map_err(ActionError::action_failed)?) +} + +async fn sic_allocate_v6_address_undo( + sagactx: ActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let ipv6 = sagactx.lookup::("allocated_control_ip")?; + Ok(nexus.free_static_v6_address(ipv6).await?) +} + +async fn sim_allocate_v6_address( + sagactx: ActionContext, +) -> Result { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let instance_id = sagactx.lookup::("instance_id")?; + Ok(nexus + .allocate_static_v6_address(instance_id) + .await + .map_err(ActionError::action_failed)?) +} + +async fn sim_allocate_v6_address_undo( + sagactx: ActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let ipv6 = sagactx.lookup::("allocated_control_ip")?; + Ok(nexus.free_static_v6_address(ipv6).await?) +} + async fn sic_instance_ensure( sagactx: ActionContext, ) -> Result<(), ActionError> { @@ -828,6 +882,15 @@ pub fn saga_instance_migrate() -> SagaTemplate { new_action_noop_undo(sim_migrate_prep), ); + template_builder.append( + "allocated_control_ip", + "AllocateIpv6ForPropolisControl", + ActionFunc::new_action( + sim_allocate_v6_address, + sim_allocate_v6_address_undo, + ), + ); + template_builder.append( "instance_migrate", "InstanceMigrate", @@ -920,6 +983,9 @@ async fn sim_instance_migrate( .await .map_err(ActionError::action_failed)?; + let allocated_control_ip = + sagactx.lookup::("allocated_control_ip")?; + let new_runtime_state: InstanceRuntimeState = dst_sa .instance_put( &instance_id, @@ -930,6 +996,7 @@ async fn sim_instance_migrate( src_propolis_addr: src_propolis_addr.to_string(), src_propolis_uuid, }), + allocated_control_ip: allocated_control_ip.to_string(), }, ) .await diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 5d3fe21f20a..443efab977b 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -630,6 +630,11 @@ "description": "Sent to a sled agent to establish the runtime state of an Instance", "type": "object", "properties": { + "allocated_control_ip": { + "description": "Allocated address for propolis's control vnic", + "type": "string", + "format": "ip" + }, "initial": { "description": "Last runtime state of the Instance known to Nexus (used if the agent has never seen this Instance before).", "allOf": [ @@ -657,6 +662,7 @@ } }, "required": [ + "allocated_control_ip", "initial", "target" ] diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 11f6d9d2eb3..7cdc53f5c46 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -100,6 +100,7 @@ async fn instance_put( body_args.initial, body_args.target, body_args.migrate, + body_args.allocated_control_ip, ) .await .map_err(Error::from)?, diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 6fa788bbcfd..d54bd1bbd57 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -195,6 +195,9 @@ struct InstanceInner { // Connection to Nexus nexus_client: Arc, + + // Allocated IP for control traffic + allocated_control_ip: std::net::IpAddr, } impl InstanceInner { @@ -367,6 +370,7 @@ mockall::mock! { initial: InstanceHardware, vlan: Option, nexus_client: Arc, + allocated_control_ip: std::net::IpAddr, ) -> Result; pub async fn start( &self, @@ -404,6 +408,7 @@ impl Instance { initial: InstanceHardware, vlan: Option, nexus_client: Arc, + allocated_control_ip: std::net::IpAddr, ) -> Result { info!(log, "Instance::new w/initial HW: {:?}", initial); let instance = InstanceInner { @@ -429,6 +434,7 @@ impl Instance { state: InstanceStates::new(initial.runtime), running_state: None, nexus_client, + allocated_control_ip, }; let inner = Arc::new(Mutex::new(instance)); @@ -483,6 +489,14 @@ impl Instance { let network = running_zone.ensure_address(AddressRequest::Dhcp).await?; info!(inner.log, "Created address {} for zone: {}", network, zname); + let v6 = running_zone + .ensure_address(AddressRequest::new_static( + inner.allocated_control_ip, + None, + )) + .await?; + info!(inner.log, "Created v6 address {} for zone: {}", v6, zname); + // Run Propolis in the Zone. let server_addr = SocketAddr::new(network.ip(), PROPOLIS_PORT); @@ -724,6 +738,7 @@ mod test { new_initial_instance(), None, Arc::new(nexus_client), + "127.0.0.1".parse().unwrap(), ) .unwrap(); diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 8cdcdf40ae6..2555a5f8608 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -81,6 +81,7 @@ impl InstanceManager { initial_hardware: InstanceHardware, target: InstanceRuntimeStateRequested, migrate: Option, + allocated_control_ip: std::net::IpAddr, ) -> Result { info!( &self.inner.log, @@ -125,6 +126,7 @@ impl InstanceManager { initial_hardware, self.inner.vlan, self.inner.nexus_client.clone(), + allocated_control_ip, )?; let instance_clone = instance.clone(); let old_instance = instances @@ -284,7 +286,7 @@ mod test { let ticket = Arc::new(std::sync::Mutex::new(None)); let ticket_clone = ticket.clone(); let instance_new_ctx = MockInstance::new_context(); - instance_new_ctx.expect().return_once(move |_, _, _, _, _, _| { + instance_new_ctx.expect().return_once(move |_, _, _, _, _, _, _| { let mut inst = MockInstance::default(); inst.expect_clone().return_once(move || { let mut inst = MockInstance::default(); @@ -313,6 +315,7 @@ mod test { migration_params: None, }, None, + "127.0.0.1".parse().unwrap(), ) .await .unwrap(); @@ -354,7 +357,7 @@ mod test { let ticket_clone = ticket.clone(); let instance_new_ctx = MockInstance::new_context(); let mut seq = mockall::Sequence::new(); - instance_new_ctx.expect().return_once(move |_, _, _, _, _, _| { + instance_new_ctx.expect().return_once(move |_, _, _, _, _, _, _| { let mut inst = MockInstance::default(); // First call to ensure (start + transition). inst.expect_clone().times(1).in_sequence(&mut seq).return_once( @@ -396,11 +399,29 @@ mod test { }; // Creates instance, start + transition. - im.ensure(id, rt.clone(), target.clone(), None).await.unwrap(); + im.ensure( + id, + rt.clone(), + target.clone(), + None, + "127.0.0.1".parse().unwrap(), + ) + .await + .unwrap(); // Transition only. - im.ensure(id, rt.clone(), target.clone(), None).await.unwrap(); + im.ensure( + id, + rt.clone(), + target.clone(), + None, + "127.0.0.1".parse().unwrap(), + ) + .await + .unwrap(); // Transition only. - im.ensure(id, rt, target, None).await.unwrap(); + im.ensure(id, rt, target, None, "127.0.0.1".parse().unwrap()) + .await + .unwrap(); assert_eq!(im.inner.instances.lock().unwrap().len(), 1); ticket.lock().unwrap().take(); diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 23c8f739a17..433a299acce 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -63,6 +63,8 @@ pub struct InstanceEnsureBody { pub target: InstanceRuntimeStateRequested, /// If we're migrating this instance, the details needed to drive the migration pub migrate: Option, + /// Allocated address for propolis's control vnic + pub allocated_control_ip: std::net::IpAddr, } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index c93074a844e..9d75ad585b4 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -198,9 +198,10 @@ impl SledAgent { initial: InstanceHardware, target: InstanceRuntimeStateRequested, migrate: Option, + allocated_control_ip: std::net::IpAddr, ) -> Result { self.instances - .ensure(instance_id, initial, target, migrate) + .ensure(instance_id, initial, target, migrate, allocated_control_ip) .await .map_err(|e| Error::Instance(e)) }