Add stat framework to Propolis

cli/src/main.rs

@@ -141,6 +141,7 @@ fn create_logger(opt: &Opt) -> Logger {
     Logger::root(drain, o!())
 }
 
+#[allow(clippy::too_many_arguments)]
 async fn new_instance(
     client: &Client,
     name: String,

client/Cargo.toml

@@ -16,4 +16,4 @@ serde_json = "1.0"
 slog = { version = "2.5", features = [ "max_level_trace", "release_max_level_debug" ] }
 thiserror = "1.0"
 uuid = { version = "1.0.0", features = [ "serde", "v4" ] }
-crucible = { git = "https://github.com/oxidecomputer/crucible", rev = "fed3e8ca7762130ee146fc516a4ef6eed2b91629" }
+crucible = { git = "https://github.com/oxidecomputer/crucible", rev = "2add0de8489f1d4de901bfe98fc28b0a6efcc3ea" }

propolis/Cargo.toml

@@ -22,7 +22,6 @@ propolis-types = { path = "../types" }
 usdt = { version = "0.3.2", default-features = false }
 tokio = { version = "1", features = ["full"] }
 futures = "0.3"
-crucible = { git = "https://github.com/oxidecomputer/crucible", rev = "fed3e8ca7762130ee146fc516a4ef6eed2b91629", optional = true }
 anyhow = "1"
 slog = "2.7"
 serde = { version = "1" }
@@ -31,6 +30,16 @@ erased-serde = "0.3"
 serde_json = "1.0"
 uuid = "1.0.0"
 
+[dependencies.crucible]
+git = "https://github.com/oxidecomputer/crucible"
+rev = "2add0de8489f1d4de901bfe98fc28b0a6efcc3ea"
+optional = true
+
+[dependencies.oximeter]
+git = "https://github.com/oxidecomputer/omicron"
+branch = "main"
+optional = true
+
 [dev-dependencies]
 crossbeam-channel = "0.5"
 tempfile = "3.2"

propolis/src/block/crucible.rs

@@ -15,6 +15,7 @@ use crucible::{
     crucible_bail, BlockIO, Buffer, CrucibleError, SnapshotDetails, Volume,
     VolumeConstructionRequest,
 };
+use oximeter::types::ProducerRegistry;
 
 /// Helper function, because Rust couldn't derive the types
 fn map_crucible_error_to_io(x: CrucibleError) -> std::io::Error {
@@ -35,21 +36,23 @@ impl CrucibleBackend {
         gen: u64,
         request: VolumeConstructionRequest,
         read_only: bool,
+        producer_registry: Option<ProducerRegistry>,
     ) -> Result<Arc<Self>> {
-        CrucibleBackend::_create(gen, request, read_only)
+        CrucibleBackend::_create(gen, request, read_only, producer_registry)
             .map_err(map_crucible_error_to_io)
     }
 
     fn _create(
         gen: u64,
         request: VolumeConstructionRequest,
         read_only: bool,
+        producer_registry: Option<ProducerRegistry>,
     ) -> anyhow::Result<Arc<Self>, crucible::CrucibleError> {
         // XXX Crucible uses std::sync::mpsc::Receiver, not
         // tokio::sync::mpsc::Receiver, so use tokio::task::block_in_place here.
         // Remove that when Crucible changes over to the tokio mpsc.
         let volume = Arc::new(tokio::task::block_in_place(|| {
-            Volume::construct(request)
+            Volume::construct(request, producer_registry)
         })?);
 
         volume.activate(gen)?;

server/Cargo.toml

@@ -22,13 +22,17 @@ async-trait = "0.1.53"
 bit_field = "0.10.1"
 bitvec = "1.0"
 bytes = "1.1"
+chrono = { version = "0.4.19", features = [ "serde" ] }
 clap = { version = "3.2", features = ["derive"] }
 const_format = "0.2"
 dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main" }
 erased-serde = "0.3"
 futures = "0.3"
 hyper =  "0.14"
 num_enum = "0.5"
+omicron-common = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
+oximeter-producer = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
+oximeter = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 ron = "0.7"
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
@@ -40,7 +44,7 @@ serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 slog = "2.7"
-propolis = { path = "../propolis", features = ["crucible"], default-features = false }
+propolis = { path = "../propolis", features = ["crucible", "oximeter"], default-features = false }
 propolis-client = { path = "../client" }
 rfb = { git = "https://github.com/oxidecomputer/rfb", branch = "main" }
 uuid = "1.0.0"

server/src/lib/initializer.rs

@@ -6,6 +6,7 @@ use std::num::NonZeroUsize;
 use std::sync::Arc;
 use std::time::SystemTime;
 
+use oximeter::types::ProducerRegistry;
 use propolis::block;
 use propolis::chardev::{self, BlockingSource, Source};
 use propolis::common::PAGE_SIZE;
@@ -123,6 +124,7 @@ pub struct MachineInitializer<'a> {
     disp: &'a Dispatcher,
     inv: &'a Inventory,
     spec: &'a InstanceSpec,
+    producer_registry: Option<ProducerRegistry>,
 }
 
 impl<'a> MachineInitializer<'a> {
@@ -133,8 +135,17 @@ impl<'a> MachineInitializer<'a> {
         disp: &'a Dispatcher,
         inv: &'a Inventory,
         spec: &'a InstanceSpec,
+        producer_registry: Option<ProducerRegistry>,
     ) -> Self {
-        MachineInitializer { log, machine, mctx, disp, inv, spec }
+        MachineInitializer {
+            log,
+            machine,
+            mctx,
+            disp,
+            inv,
+            spec,
+            producer_registry,
+        }
     }
 
     pub fn initialize_rom<P: AsRef<std::path::Path>>(
@@ -279,6 +290,7 @@ impl<'a> MachineInitializer<'a> {
                         )
                     })?,
                     backend_spec.readonly,
+                    self.producer_registry.clone(),
                 )?;
                 let child = inventory::ChildRegister::new(
                     &be,

server/src/lib/lib.rs

@@ -6,4 +6,5 @@ mod migrate;
 mod serial;
 pub mod server;
 mod spec;
+pub mod stats;
 pub mod vnc;

server/src/lib/server.rs

@@ -14,6 +14,7 @@ use propolis::hw::qemu::ramfb::RamFb;
 use rfb::server::VncServer;
 use slog::{error, info, o, Logger};
 use std::collections::BTreeMap;
+use std::net::SocketAddr;
 use std::ops::Range;
 use std::sync::Arc;
 use thiserror::Error;
@@ -32,6 +33,7 @@ use crate::config::Config;
 use crate::initializer::{build_instance, MachineInitializer};
 use crate::serial::Serial;
 use crate::spec::SpecBuilder;
+use crate::stats::{prop_oximeter, PropCountStat, PropStatOuter};
 use crate::vnc::PropolisVncServer;
 use crate::{migrate, vnc};
 use uuid::Uuid;
@@ -93,6 +95,17 @@ pub(crate) struct InstanceContext {
     pub(crate) crucible_backends: Mutex<CrucibleBackendMap>,
 }
 
+#[derive(Debug, Clone)]
+pub struct InstanceMetricsConfig {
+    pub propolis_addr: SocketAddr,
+    pub metric_addr: SocketAddr,
+}
+impl InstanceMetricsConfig {
+    pub fn new(propolis_addr: SocketAddr, metric_addr: SocketAddr) -> Self {
+        InstanceMetricsConfig { propolis_addr, metric_addr }
+    }
+}
+
 /// Contextual information accessible from HTTP callbacks.
 pub struct Context {
     pub(crate) context: Mutex<Option<InstanceContext>>,
@@ -101,6 +114,9 @@ pub struct Context {
     log: Logger,
     pub(crate) vnc_server: Arc<Mutex<VncServer<PropolisVncServer>>>,
     pub(crate) use_reservoir: bool,
+    // To register with Oximeter.
+    pub(crate) metric_config: Option<InstanceMetricsConfig>,
+    pub instance_metrics: Mutex<Option<PropStatOuter>>,
 }
 
 impl Context {
@@ -110,6 +126,7 @@ impl Context {
         vnc_server: VncServer<PropolisVncServer>,
         use_reservoir: bool,
         log: Logger,
+        metric_config: Option<InstanceMetricsConfig>,
     ) -> Self {
         Context {
             context: Mutex::new(None),
@@ -118,6 +135,8 @@ impl Context {
             log,
             vnc_server: Arc::new(Mutex::new(vnc_server)),
             use_reservoir,
+            metric_config,
+            instance_metrics: Mutex::new(None),
         }
     }
 }
@@ -206,6 +225,67 @@ async fn instance_ensure(
         }));
     }
 
+    // If anyone outside Propolis wishes to register for metrics, this
+    // will hold the producer registry they can use.
+    let mut producer_registry = None;
+
+    // Determine if we need to setup the metrics endpoint or not.
+    // If we do, we will then populate producer_registry with something.
+    if server_context.metric_config.is_some() {
+        // Create some propolis level metrics.
+        let prop_count_stat = PropCountStat::new(properties.id.clone());
+        let pso = PropStatOuter {
+            prop_stat_wrap: Arc::new(std::sync::Mutex::new(prop_count_stat)),
+        };
+
+        // This is the address where stats will be collected.
+        let propolis_addr =
+            server_context.metric_config.as_ref().unwrap().propolis_addr.ip();
+        let listen_addr = SocketAddr::new(propolis_addr, 0);
+        let register_addr =
+            server_context.metric_config.as_ref().unwrap().metric_addr;
+
+        match prop_oximeter(
+            properties.id.clone(),
+            listen_addr,
+            register_addr,
+            rqctx.log.clone(),
+        )
+        .await
+        {
+            Err(e) => {
+                error!(rqctx.log, "Failed to register with Oximeter {:?}", e);
+            }
+            Ok(server) => {
+                info!(
+                    rqctx.log,
+                    "registering metrics with instance uuid: {}", properties.id,
+                );
+                // Register the propolis level instance metrics.
+                server.registry().register_producer(pso.clone()).unwrap();
+
+                // Now that our metrics are registered, attach them to
+                // the server context so they can be updated.
+                let mut im = server_context.instance_metrics.lock().await;
+                *im = Some(pso.clone());
+                drop(im);
+
+                // Clone the producer_registry that we can pass to any
+                // other library that may want to register their own
+                // metrics.  Doing it this way means propolis does not have
+                // to know what metrics they register.
+                producer_registry = Some(server.registry().clone());
+
+                // Spawn the metric endpoint.
+                tokio::spawn(async move {
+                    server.serve_forever().await.unwrap();
+                });
+            }
+        }
+    } else {
+        info!(rqctx.log, "No metrics registration was requested");
+    }
+
     let mut in_memory_disk_contents: BTreeMap<String, Vec<u8>> =
         BTreeMap::new();
     let mut spec_builder =
@@ -313,6 +393,7 @@ async fn instance_ensure(
                 disp,
                 inv,
                 &spec,
+                producer_registry.clone(),
             );
             init.initialize_rom(server_context.config.get_bootrom())?;
             init.initialize_kernel_devs()?;
@@ -482,6 +563,15 @@ async fn instance_state_put(
         HttpError::for_internal_error(format!("Failed to set state: {:?}", err))
     })?;
 
+    // Update the metrics counter when we apply a reset
+    if state == propolis::instance::ReqState::Reset {
+        let server_context = rqctx.context();
+        let instance_metrics = server_context.instance_metrics.lock().await;
+        if let Some(im) = &*instance_metrics {
+            im.count_reset();
+        }
+    }
+
     Ok(HttpResponseUpdatedNoContent {})
 }