Tofino observability features

.github/buildomat/common.sh

@@ -11,9 +11,9 @@
 TOFINO_STAGES=20
 
 # These describe which version of the SDE to download and where to find it
-SDE_COMMIT=e61fe02c3c1c384b2e212c90177fcea76a31fd4e
-SDE_PKG_SHA256=8a87a9b0bed3c5440a173a7a41361bdeb5e7a848882da6b4aa48c8fb0043f3bd
-SDE_DEB_SHA256=a292e2dd5311647c4852bb41f2532dd1fdf30325b6d04cccb7e85b873e521d5f
+SDE_COMMIT=53519b8cf74fe832cc7838ea92683564ce4026f2
+SDE_PKG_SHA256=ed783a1e7c8d59c392e8cc89114fb0d495b5475373b762068a719e0fb215f5a0
+SDE_DEB_SHA256=90a18b65a6c65f4d15d5f75a00e42ae55a27ffaff2066061aa95feefbe85e163
 
 [ `uname -s` == "SunOS" ] && SERIES=illumos
 [ `uname -s` == "SunOS" ] || SERIES=linux

.github/buildomat/packet-test-common.sh

@@ -53,7 +53,8 @@ if [[ $JUST_TEST -ne 1 ]]; then
         libssl-dev \
         pkg-config \
         libcli-dev \
-        sysvbanner
+        sysvbanner \
+        libboost-all-dev
 fi
 
 export SDE=/opt/oxide/tofino_sde

.helix/languages.toml

@@ -4,4 +4,4 @@ name = "rust"
 # default to tofino_sde, can change this to any other feature such as softnpu
 # as needed during development but in source control this should probably
 # remain as tofino_sde
-features = [ "tofino_sde" ]
+features = [ "tofino_asic" ]

aal/src/lib.rs

@@ -285,6 +285,7 @@ pub trait TableOps<H: AsicOps> {
     fn get_entries<M: MatchParse, A: ActionParse>(
         &self,
         hdl: &H,
+        from_hardware: bool,
     ) -> AsicResult<Vec<(M, A)>>;
     fn get_counters<M: MatchParse>(
         &self,

asic/build.rs

@@ -37,6 +37,7 @@ fn gen_bindings(sde_dir: &str) -> Result<()> {
         format!("{sde_includes}/bf_rt/bf_rt_session.h"),
         format!("{sde_includes}/lld/lld_sku.h"),
         format!("{sde_includes}/tofino/bf_pal/bf_pal_port_intf.h"),
+        format!("{sde_includes}/pipe_mgr/pipe_mgr_intf.h"),
     ];
 
     // There are several headers we only use (and _can_ only use) when running

asic/src/chaos/table.rs

@@ -137,6 +137,7 @@ impl TableOps<Handle> for Table {
     fn get_entries<M: MatchParse, A: ActionParse>(
         &self,
         _hdl: &Handle,
+        _from_hardware: bool,
     ) -> AsicResult<Vec<(M, A)>> {
         Err(AsicError::OperationUnsupported)
     }

asic/src/softnpu/table.rs

@@ -616,9 +616,11 @@ impl TableOps<Handle> for Table {
     fn get_entries<M: MatchParse, A: ActionParse>(
         &self,
         _hdl: &Handle,
+        _from_hardware: bool,
     ) -> AsicResult<Vec<(M, A)>> {
         Err(aal::AsicError::OperationUnsupported)
     }
+
     fn get_counters<M: MatchParse>(
         &self,
         _hdl: &Handle,

asic/src/tofino_asic/c/bf_wrapper.h

@@ -43,4 +43,46 @@ struct driver_version {
 	uint32_t patch;
 };
 
+bf_status_t
+lld_enable_all_ints(
+  bf_dev_id_t dev_id,
+  bf_subdev_id_t subdev_id
+);
+
+bf_status_t
+lld_dump_new_ints(
+  bf_dev_id_t dev_id,
+  bf_subdev_id_t subdev_id
+);
+
+bf_status_t
+lld_int_poll(
+  bf_dev_id_t dev_id,
+  bf_subdev_id_t subdev_id,
+  bool all_ints
+);
+
+bf_status_t
+bf_err_interrupt_handling_mode_set(
+  bf_dev_id_t dev_id,
+  bool enable
+);
+
+pipe_status_t
+pipe_mgr_tcam_scrub_timer_set(
+  bf_dev_id_t dev,
+  uint32_t msec_timer
+);
+
+uint32_t
+pipe_mgr_tcam_scrub_timer_get(
+  bf_dev_id_t dev
+);
+
+bool
+pipe_mgr_is_device_locked(
+  bf_dev_id_t dev_id
+);
+
+
 #endif /* BF_WRAPPER_H */

asic/src/tofino_asic/imported_bf_functions

@@ -160,3 +160,34 @@ bf_pm_fsm_transition_callback
 bf_sys_log_callback
 
 lld_sku_map_dev_port_id_to_mac_ch
+
+# Snapshot
+bf_snapshot_create
+bf_snapshot_delete
+bf_snapshot_clear
+bf_snapshot_state_set
+bf_snapshot_state_get
+bf_snapshot_cfg_set
+bf_snapshot_capture_trigger_field_add
+bf_snapshot_capture_trigger_field_add_bytes
+bf_snapshot_capture_trigger_fields_clr
+bf_snapshot_capture_phv_fields_dict_size
+bf_snapshot_capture_get
+bf_snapshot_capture_decode_field_value
+bf_snapshot_capture_decode_field_value_bytes
+bf_snapshot_raw_capture_get
+bf_snapshot_total_phv_count_get
+bf_snapshot_field_in_scope
+bf_snapshot_trigger_field_in_scope
+bf_snapshot_do_polling
+bf_snapshot_entry_params_get
+bf_snapshot_handle_get
+
+# Interrupt management
+lld_enable_all_ints
+lld_dump_new_ints
+lld_int_poll
+bf_err_interrupt_handling_mode_set
+pipe_mgr_is_device_locked
+pipe_mgr_tcam_scrub_timer_set
+pipe_mgr_tcam_scrub_timer_get

asic/src/tofino_asic/imported_bf_types

@@ -5,3 +5,7 @@ pm_intf_fsm_states_t
 bf_port_ber_t
 qsfp_fsm_state_t
 qsfp_fsm_ch_en_state_t
+bf_snapshot_capture_ctrl_info_t
+bf_snapshot_tables_info_t
+bf_snapshot_capture_ctrl_info_arr_t
+bf_snapshot_trigger_field_t

asic/src/tofino_asic/interrupt_monitor.rs

@@ -0,0 +1,131 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/
+//
+// Copyright 2026 Oxide Computer Company
+
+use std::{thread::sleep, time::Duration};
+
+use crate::tofino_asic::{
+    BF_INVALID_ARG, BF_SUCCESS,
+    bf_wrapper::bf_error_str,
+    genpd::{
+        bf_dev_id_t, bf_err_interrupt_handling_mode_set, bf_subdev_id_t,
+        lld_dump_new_ints, lld_enable_all_ints, lld_int_poll,
+        pipe_mgr_is_device_locked, pipe_mgr_tcam_scrub_timer_set,
+    },
+};
+use slog::{Logger, info, warn};
+
+const DEV_ID: bf_dev_id_t = 0;
+const SUBDEV_ID: bf_subdev_id_t = 0;
+const INTERVAL: Duration = Duration::from_secs(5);
+
+/// Monitoring interrupts requires a number of precursory steps to set things up
+/// in the SDE and on the ASIC. The `monitor_interrupts` function is designed as
+/// a state machine that will drive forward toward the termial state of actively
+/// running the monitoring loop.
+///
+/// There are a number of errors that can happen in the SDE in the precursory
+/// states. The code in the SDE is pretty twisty and the error model is not
+/// totally clear, but it does appear to me that some of the errors that can
+/// happen are transient in nature. In particular errors associated with timer
+/// initialization and locking. With that in mind, each precursory state is
+/// a loop that will go forever until the required SDE functions have been
+/// called successfully. The loops have a 5 second interval to avoid excessive
+/// iteration.
+///
+/// The idea here is to get to the terminal state as soon as possible. Because
+/// the terminal state is an infinite loop, it affords us the opportunity to
+/// make each precursory loop run until success.
+pub fn monitor_interrupts(log: Logger) -> ! {
+    let log = log.new(slog::o!("unit" => "interrupt monitor"));
+    info!(log, "starring interrupt monitor");
+
+    wait_for_unlock(&log);
+    enable_tcam_scrub(&log);
+    set_interrupt_handling_mode(&log);
+    enable_lld_interrupts(&log);
+
+    monitoring_loop(&log)
+}
+
+fn wait_for_unlock(log: &Logger) {
+    loop {
+        if unsafe { pipe_mgr_is_device_locked(DEV_ID) } {
+            warn!(log, "asic is locked, cannot start");
+            sleep(INTERVAL);
+            continue;
+        }
+        info!(log, "asic is unlocked - starting interrupt monitor");
+        break;
+    }
+}
+
+fn enable_tcam_scrub(log: &Logger) {
+    loop {
+        // This value is in milliseconds so this is 2 minutes. This is the
+        // default SDE value.
+        let rc = unsafe { pipe_mgr_tcam_scrub_timer_set(DEV_ID, 120000) };
+        if rc == BF_SUCCESS {
+            info!(log, "tcam scrub set to 2 minute interval");
+            break;
+        }
+        warn!(
+            log,
+            "failed to enable tcam scrub";
+            "error" => bf_error_str(rc)
+        );
+        sleep(INTERVAL)
+    }
+}
+
+fn set_interrupt_handling_mode(log: &Logger) {
+    loop {
+        let rc = unsafe { bf_err_interrupt_handling_mode_set(DEV_ID, true) };
+        if rc == BF_SUCCESS {
+            info!(log, "interrupt handling mode set");
+            break;
+        }
+        warn!(
+            log,
+            "failed to set interrupt handling mode";
+            "error" => bf_error_str(rc)
+        );
+        sleep(INTERVAL);
+    }
+}
+
+fn enable_lld_interrupts(log: &Logger) {
+    loop {
+        let rc = unsafe { lld_enable_all_ints(DEV_ID, SUBDEV_ID) };
+        if rc == BF_SUCCESS {
+            info!(log, "enabled lld interrupts");
+            break;
+        }
+        warn!(
+            log,
+            "failed to enable lld interrupts";
+            "error" => bf_error_str(rc)
+        );
+        sleep(INTERVAL);
+    }
+}
+
+fn monitoring_loop(log: &Logger) -> ! {
+    loop {
+        let rc = unsafe { lld_int_poll(DEV_ID, SUBDEV_ID, true) };
+        if rc != BF_SUCCESS {
+            warn!(log, "lld_int_poll: {}", bf_error_str(rc));
+        }
+
+        let rc = unsafe { lld_dump_new_ints(DEV_ID, SUBDEV_ID) };
+        // BF_INVALID_ARG means interrupts were found and dumped (yes, really)
+        // BF_SUCCESS means no new interrupts
+        // Anything else is an actual error
+        if rc != BF_SUCCESS && rc != BF_INVALID_ARG {
+            warn!(log, "lld_dump_new_ints: {}", bf_error_str(rc));
+        }
+        sleep(INTERVAL);
+    }
+}

asic/src/tofino_asic/mod.rs

@@ -17,13 +17,15 @@ use common::ports::*;
 mod bf_wrapper;
 mod genpd;
 
+pub mod interrupt_monitor;
 mod link_fsm;
 #[cfg(feature = "multicast")]
 pub mod mcast;
 pub mod ports;
 pub mod qsfp;
 mod sde_log;
 pub mod serdes;
+pub mod snapshot;
 pub mod stats;
 pub mod table;