From e42851cb610c8572ae0253a1c50f857ca97a9cdb Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 7 Jul 2026 14:25:56 -0600 Subject: [PATCH 01/14] feat(observability): project marks as tool spans Signed-off-by: Bryan Bednarski --- crates/core/src/observability/atif.rs | 145 ++++++++++++++--- crates/core/src/observability/mod.rs | 29 ++++ .../core/src/observability/openinference.rs | 96 +++++++++++- crates/core/src/observability/otel.rs | 83 +++++++++- .../src/observability/plugin_component.rs | 30 +++- crates/core/tests/unit/atif_tests.rs | 78 +++++++++ .../unit/observability/openinference_tests.rs | 127 +++++++++++++++ .../tests/unit/observability/otel_tests.rs | 148 ++++++++++++++++++ .../observability/plugin_component_tests.rs | 85 ++++++++-- crates/node/observability.d.ts | 2 + crates/node/observability.js | 2 + .../node/tests/observability_plugin_tests.mjs | 4 + docs/configure-plugins/observability/atif.mdx | 7 + .../observability/openinference.mdx | 7 + .../observability/opentelemetry.mdx | 8 + go/nemo_relay/observability_plugin.go | 34 ++-- go/nemo_relay/observability_plugin_test.go | 10 +- python/nemo_relay/observability.py | 7 + python/nemo_relay/observability.pyi | 4 + python/tests/test_observability_plugin.py | 4 + 20 files changed, 854 insertions(+), 56 deletions(-) diff --git a/crates/core/src/observability/atif.rs b/crates/core/src/observability/atif.rs index ce96140e9..8dd78820f 100644 --- a/crates/core/src/observability/atif.rs +++ b/crates/core/src/observability/atif.rs @@ -22,7 +22,7 @@ //! | LLM End | `agent` step | Response content, tool_calls promoted| //! | Tool Start | *(skipped)* | tool_calls come from LLM End instead | //! | Tool End | agent observation | Correlated by `source_call_id` | -//! | Mark (with data)| `system` step | Custom event data preserved | +//! | Mark (with data)| `system` step | Optional deterministic tool projection | //! | Scope Start/End | *(skipped)* | Structural events, not trajectory | //! //! The exporter serializes the full collected event stream into a single ATIF @@ -43,7 +43,10 @@ use crate::codec::response::AnnotatedLlmResponse; use crate::error::Result; use crate::json::Json; -use super::{estimate_cost_for_response_or_model, manual, merge_usage, model_name_for_llm_event}; +use super::{ + MarkProjection, estimate_cost_for_response_or_model, is_llm_chunk_mark, manual, merge_usage, + model_name_for_llm_event, +}; /// The ATIF schema version string embedded in all exported trajectories. /// @@ -276,6 +279,12 @@ pub struct AtifStepExtra { /// Full raw point-in-time event payload for mark/system steps. #[serde(skip_serializing_if = "Option::is_none")] pub event_payload: Option, + /// Canonical ATOF category for mark/system steps. + #[serde(skip_serializing_if = "Option::is_none")] + pub event_category: Option, + /// Canonical ATOF category profile for mark/system steps. + #[serde(skip_serializing_if = "Option::is_none")] + pub event_category_profile: Option, /// Per-tool callable lineage, aligned with `tool_calls`. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub tool_ancestry: Vec, @@ -322,6 +331,7 @@ pub struct AtifTrajectory { struct AtifExporterState { session_id: String, agent_info: AtifAgentInfo, + mark_projection: MarkProjection, events: Vec, } @@ -348,11 +358,22 @@ impl AtifExporter { state: Arc::new(Mutex::new(AtifExporterState { session_id, agent_info, + mark_projection: MarkProjection::default(), events: Vec::new(), })), } } + /// Selects how point-in-time marks are represented in ATIF output. + /// + /// The default [`MarkProjection::Event`] preserves marks as system steps. + /// [`MarkProjection::Tool`] creates deterministic agent/tool steps for + /// consumers that visualize tool calls but do not render system events. + pub fn with_mark_projection(self, mark_projection: MarkProjection) -> Self { + self.state.lock().unwrap().mark_projection = mark_projection; + self + } + /// Return an event subscriber function that records NeMo Relay events. /// /// The returned callback can be registered with @@ -392,11 +413,12 @@ impl AtifExporter { /// callers that prefer an explicitly fallible method name. pub fn try_export(&self) -> Result { flush_subscribers()?; - let (session_id, agent_info, events) = { + let (session_id, agent_info, mark_projection, events) = { let state = self.state.lock().unwrap(); ( state.session_id.clone(), state.agent_info.clone(), + state.mark_projection, state.events.clone(), ) }; @@ -404,6 +426,7 @@ impl AtifExporter { Ok(events_to_trajectory( &session_id, agent_info, + mark_projection, &collected_events, )) } @@ -1888,6 +1911,8 @@ impl PendingAgentStep { llm_request: None, llm_response: self.llm_response.take(), event_payload: None, + event_category: None, + event_category_profile: None, tool_ancestry: std::mem::take(&mut self.tool_ancestry), tool_invocations: if self.tool_invocations.is_empty() { None @@ -1965,6 +1990,7 @@ impl PendingAgentStep { #[derive(Default)] struct StepConversionState { + mark_projection: MarkProjection, steps: Vec, last_tool_call_map: std::collections::HashMap, tool_scope_call_ids: std::collections::HashMap, @@ -2244,6 +2270,8 @@ impl StepConversionState { llm_request: Some(content.clone()), llm_response: None, event_payload: None, + event_category: None, + event_category_profile: None, tool_ancestry: Vec::new(), tool_invocations: None, }; @@ -2598,6 +2626,10 @@ impl StepConversionState { if is_empty_mark_payload(data) { return; } + if self.mark_projection == MarkProjection::Tool { + self.handle_mark_as_tool(mark, lookups, data); + return; + } let extra = AtifStepExtra { ancestry: build_ancestry(mark, &lookups.name_map), invocation: Some(AtifInvocationInfo { @@ -2610,6 +2642,12 @@ impl StepConversionState { llm_request: None, llm_response: None, event_payload: Some(data.clone()), + event_category: mark + .category() + .map(|category| category.as_str().to_string()), + event_category_profile: mark + .category_profile() + .and_then(|profile| serde_json::to_value(profile).ok()), tool_ancestry: Vec::new(), tool_invocations: None, }; @@ -2630,6 +2668,67 @@ impl StepConversionState { }); } + fn handle_mark_as_tool(&mut self, mark: &Event, lookups: &EventLookupMaps, data: &Json) { + self.finalize_agent_extra(); + + let ancestry = build_ancestry(mark, &lookups.name_map); + let invocation = AtifInvocationInfo { + start_timestamp: None, + end_timestamp: None, + invocation_id: Some(mark.uuid().to_string()), + status: Some("completed".to_string()), + framework: Some("nemo_relay".to_string()), + }; + let source_call_id = format!("mark:{}", mark.uuid()); + let mut observation_extra = event_extra(mark); + if let Json::Object(extra) = &mut observation_extra { + extra.insert("event_payload".to_string(), data.clone()); + } + let extra = AtifStepExtra { + ancestry: ancestry.clone(), + invocation: Some(invocation.clone()), + llm_request: None, + llm_response: None, + event_payload: Some(data.clone()), + event_category: mark + .category() + .map(|category| category.as_str().to_string()), + event_category_profile: mark + .category_profile() + .and_then(|profile| serde_json::to_value(profile).ok()), + tool_ancestry: vec![ancestry], + tool_invocations: Some(vec![invocation]), + }; + + self.steps.push(AtifStep { + step_id: 0, + source: "agent".to_string(), + message: empty_message(), + timestamp: Some(mark.timestamp().to_rfc3339()), + model_name: None, + reasoning_effort: None, + reasoning_content: None, + tool_calls: Some(vec![AtifToolCall { + tool_call_id: source_call_id.clone(), + function_name: mark.name().to_string(), + arguments: data.clone(), + extra: Some(event_extra(mark)), + }]), + observation: Some(AtifObservation { + results: vec![AtifObservationResult { + source_call_id: Some(source_call_id), + content: Some(mark_message(mark, data)), + subagent_trajectory_ref: None, + extra: Some(observation_extra), + }], + }), + metrics: None, + llm_call_count: Some(0), + is_copied_context: None, + extra: serde_json::to_value(&extra).ok(), + }); + } + fn handle_subagent_start(&mut self, child: &AgentScopeNode, event: &Event) { let source_call_id = self.resolve_subagent_source_call_id(event); self.flush_observations(); @@ -3171,6 +3270,7 @@ fn nearest_non_turn_agent_parent( fn events_to_trajectory( session_id: &str, agent_info: AtifAgentInfo, + mark_projection: MarkProjection, events: &[&Event], ) -> AtifTrajectory { let mut sorted: Vec<&Event> = events.to_vec(); @@ -3180,10 +3280,18 @@ fn events_to_trajectory( if let Some(root_uuid) = tree.choose_root(session_id) && can_use_agent_scope_tree(&tree, &sorted) { - return agent_scope_to_trajectory(&tree, root_uuid, session_id, &agent_info, &sorted, true); + return agent_scope_to_trajectory( + &tree, + root_uuid, + session_id, + &agent_info, + mark_projection, + &sorted, + true, + ); } - let steps = events_to_steps(&sorted); + let steps = events_to_steps(&sorted, mark_projection); trajectory_from_parts( session_id.to_string(), Some(session_id.to_string()), @@ -3227,10 +3335,11 @@ fn agent_scope_to_trajectory( agent_uuid: Uuid, session_id: &str, agent_info: &AtifAgentInfo, + mark_projection: MarkProjection, sorted_events: &[&Event], is_root: bool, ) -> AtifTrajectory { - let mut steps = events_to_steps_for_agent(sorted_events, tree, agent_uuid); + let mut steps = events_to_steps_for_agent(sorted_events, tree, agent_uuid, mark_projection); let subagent_trajectories = tree .nodes .get(&agent_uuid) @@ -3243,6 +3352,7 @@ fn agent_scope_to_trajectory( *child_uuid, session_id, agent_info, + mark_projection, sorted_events, false, ) @@ -3316,9 +3426,13 @@ fn events_to_steps_for_agent( events: &[&Event], tree: &AgentScopeTree, agent_uuid: Uuid, + mark_projection: MarkProjection, ) -> Vec { let lookups = EventLookupMaps::from_events_for_agent(events, tree, agent_uuid); - let mut state = StepConversionState::default(); + let mut state = StepConversionState { + mark_projection, + ..StepConversionState::default() + }; for event in events { if let Some(child) = tree.direct_child_for_start(agent_uuid, event) { @@ -3354,11 +3468,14 @@ fn events_to_steps_for_agent( /// promoted tool_calls by function name → `source_call_id`. /// 5. Mark events → system steps if they carry data. /// 6. Scope Start/End → skipped. -fn events_to_steps(events: &[&Event]) -> Vec { +fn events_to_steps(events: &[&Event], mark_projection: MarkProjection) -> Vec { let mut sorted: Vec<&Event> = events.to_vec(); sorted.sort_by_key(|e| *e.timestamp()); let lookups = EventLookupMaps::from_events(&sorted); - let mut state = StepConversionState::default(); + let mut state = StepConversionState { + mark_projection, + ..StepConversionState::default() + }; for event in &sorted { state.handle_event(event, &lookups); @@ -3371,16 +3488,6 @@ fn is_empty_mark_payload(data: &Json) -> bool { data.is_null() || data.as_object().is_some_and(|object| object.is_empty()) } -fn is_llm_chunk_mark(mark: &Event) -> bool { - mark.name() == "llm.chunk" - || mark - .metadata() - .and_then(Json::as_object) - .and_then(|metadata| metadata.get("hook_event_name")) - .and_then(Json::as_str) - == Some("llm.chunk") -} - // A runtime mark is point-in-time telemetry rather than a scoped call with start/end events. Agent // hook adapters use marks for lifecycle notifications that do not map to first-class ATIF step // types, for example hook-only status updates or synthetic fallback events. The ATIF step message diff --git a/crates/core/src/observability/mod.rs b/crates/core/src/observability/mod.rs index fd35def8d..122f8615a 100644 --- a/crates/core/src/observability/mod.rs +++ b/crates/core/src/observability/mod.rs @@ -4,6 +4,7 @@ //! Optional observability integrations for NeMo Relay Core. use crate::api::event::EventNormalizationExt; +use serde::{Deserialize, Serialize}; #[cfg(test)] use std::sync::Mutex; @@ -22,6 +23,34 @@ pub mod openinference; pub mod otel; pub mod plugin_component; +/// Export representation for point-in-time mark events. +/// +/// Marks remain canonical ATOF events regardless of this setting. Exporters +/// apply the selected projection only when translating those events into a +/// downstream trajectory or trace format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum MarkProjection { + /// Preserve marks as ATIF system steps or OTEL span events. + #[default] + Event, + /// Render non-streaming marks as deterministic ATIF tool steps or + /// zero-duration OTEL child spans so trace-tree consumers can display them + /// directly. High-volume `llm.chunk` marks remain events. + Tool, +} + +pub(crate) fn is_llm_chunk_mark(event: &crate::api::event::Event) -> bool { + event.name() == "llm.chunk" + || event + .metadata() + .and_then(crate::json::Json::as_object) + .and_then(|metadata| metadata.get("hook_event_name")) + .and_then(crate::json::Json::as_str) + == Some("llm.chunk") +} + #[cfg(all(test, feature = "otel", feature = "openinference"))] #[path = "../../tests/unit/observability/exporter_parity_tests.rs"] mod exporter_parity_tests; diff --git a/crates/core/src/observability/openinference.rs b/crates/core/src/observability/openinference.rs index 649460c96..90de8b65a 100644 --- a/crates/core/src/observability/openinference.rs +++ b/crates/core/src/observability/openinference.rs @@ -7,7 +7,7 @@ //! //! - scope/tool/LLM `Start` events open spans //! - matching `End` events close spans -//! - `Mark` events become span events on the active parent span when possible +//! - `Mark` events become span events by default, with an optional OpenInference tool projection //! - orphan marks fall back to zero-duration spans so they still reach OTLP //! //! The public API is intentionally small: @@ -21,8 +21,9 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use super::{ - estimate_cost_for_response_or_model, estimate_cost_for_response_or_requested_model, manual, - merge_usage, model_name_for_llm_event, + MarkProjection, estimate_cost_for_response_or_model, + estimate_cost_for_response_or_requested_model, is_llm_chunk_mark, manual, merge_usage, + model_name_for_llm_event, }; use crate::api::event::{Event, EventNormalizationExt, ScopeCategory}; use crate::api::runtime::EventSubscriberFn; @@ -101,6 +102,7 @@ pub struct OpenInferenceConfig { service_namespace: Option, service_version: Option, instrumentation_scope: String, + mark_projection: MarkProjection, timeout: Duration, transport: OtlpTransport, } @@ -115,6 +117,7 @@ impl Default for OpenInferenceConfig { service_namespace: None, service_version: None, instrumentation_scope: "nemo-relay-openinference".to_string(), + mark_projection: MarkProjection::default(), timeout: Duration::from_secs(3), transport: OtlpTransport::HttpBinary, } @@ -184,6 +187,12 @@ impl OpenInferenceConfig { self.instrumentation_scope = scope.into(); self } + + /// Selects how point-in-time marks are represented in exported traces. + pub fn with_mark_projection(mut self, mark_projection: MarkProjection) -> Self { + self.mark_projection = mark_projection; + self + } } /// OpenInference-backed NeMo Relay subscriber. @@ -209,6 +218,7 @@ impl OpenInferenceSubscriber { Ok(Self::from_tracer_provider_with_scope( provider, config.instrumentation_scope, + config.mark_projection, )) } @@ -217,17 +227,38 @@ impl OpenInferenceSubscriber { provider: SdkTracerProvider, instrumentation_scope: impl Into, ) -> Self { - Self::from_tracer_provider_with_scope(provider, instrumentation_scope.into()) + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + MarkProjection::default(), + ) + } + + /// Builds a subscriber from a tracer provider with an explicit mark projection. + pub fn from_tracer_provider_with_mark_projection( + provider: SdkTracerProvider, + instrumentation_scope: impl Into, + mark_projection: MarkProjection, + ) -> Self { + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + mark_projection, + ) } fn from_tracer_provider_with_scope( provider: SdkTracerProvider, instrumentation_scope: String, + mark_projection: MarkProjection, ) -> Self { - let processor = Arc::new(Mutex::new(OpenInferenceEventProcessor::new( - provider, - instrumentation_scope, - ))); + let processor = Arc::new(Mutex::new( + OpenInferenceEventProcessor::new_with_mark_projection( + provider, + instrumentation_scope, + mark_projection, + ), + )); let processor_for_callback = Arc::clone(&processor); let subscriber: EventSubscriberFn = Arc::new(move |event: &Event| { let Ok(mut guard) = processor_for_callback.lock() else { @@ -381,10 +412,20 @@ struct OpenInferenceEventProcessor { completed_span_order: VecDeque, provider: SdkTracerProvider, tracer: SdkTracer, + mark_projection: MarkProjection, } impl OpenInferenceEventProcessor { + #[cfg(test)] fn new(provider: SdkTracerProvider, instrumentation_scope: String) -> Self { + Self::new_with_mark_projection(provider, instrumentation_scope, MarkProjection::default()) + } + + fn new_with_mark_projection( + provider: SdkTracerProvider, + instrumentation_scope: String, + mark_projection: MarkProjection, + ) -> Self { let tracer = provider.tracer(instrumentation_scope); Self { active_spans: HashMap::new(), @@ -392,6 +433,7 @@ impl OpenInferenceEventProcessor { completed_span_order: VecDeque::new(), provider, tracer, + mark_projection, } } @@ -442,6 +484,10 @@ impl OpenInferenceEventProcessor { } fn process_mark(&mut self, event: &Event) { + if self.mark_projection == MarkProjection::Tool && !is_llm_chunk_mark(event) { + self.process_mark_as_tool(event); + return; + } let mark_name = event.name().to_string(); let timestamp = to_system_time(*event.timestamp()); let attributes = mark_attributes(event); @@ -469,6 +515,29 @@ impl OpenInferenceEventProcessor { span.end_with_timestamp(timestamp); } + fn process_mark_as_tool(&mut self, event: &Event) { + let timestamp = to_system_time(*event.timestamp()); + let orphan = self.find_parent_span(event).is_none(); + let mut attributes = mark_attributes(event); + attributes.push(KeyValue::new("nemo_relay.mark.projection", "tool")); + attributes.push(KeyValue::new( + oi::OPENINFERENCE_SPAN_KIND, + OpenInferenceSpanKind::Tool, + )); + if orphan { + attributes.push(KeyValue::new("nemo_relay.mark.orphan", true)); + } + + let mut span = self + .tracer + .span_builder(format!("mark:{}", event.name())) + .with_kind(SpanKind::Internal) + .with_start_time(timestamp) + .start_with_context(&self.tracer, &self.parent_context(event)); + span.set_attributes(attributes); + span.end_with_timestamp(timestamp); + } + fn parent_context(&self, event: &Event) -> Context { if let Some(active_span) = self.find_parent_span(event) { return Context::new().with_remote_span_context(active_span.span_context.clone()); @@ -1108,6 +1177,17 @@ fn mark_attributes(event: &Event) -> Vec { "nemo_relay.mark.metadata_json", event.metadata(), ); + if let Some(category) = event.category() { + attributes.push(KeyValue::new( + "nemo_relay.mark.category", + category.as_str().to_string(), + )); + } + push_serialized( + &mut attributes, + "nemo_relay.mark.category_profile_json", + event.category_profile(), + ); attributes } diff --git a/crates/core/src/observability/otel.rs b/crates/core/src/observability/otel.rs index 474af253b..1bc6b2b24 100644 --- a/crates/core/src/observability/otel.rs +++ b/crates/core/src/observability/otel.rs @@ -7,7 +7,7 @@ //! //! - scope/tool/LLM `Start` events open spans //! - matching `End` events close spans -//! - `Mark` events become span events on the active parent span when possible +//! - `Mark` events become span events by default, with an optional visible child-span projection //! - orphan marks fall back to zero-duration spans so they still reach OTLP //! //! The public API is intentionally small: @@ -21,7 +21,8 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use super::{ - estimate_cost_for_response_or_model, estimate_cost_for_response_or_requested_model, manual, + MarkProjection, estimate_cost_for_response_or_model, + estimate_cost_for_response_or_requested_model, is_llm_chunk_mark, manual, model_name_for_llm_event, }; use crate::api::event::{Event, EventNormalizationExt, ScopeCategory}; @@ -95,6 +96,7 @@ pub struct OpenTelemetryConfig { service_namespace: Option, service_version: Option, instrumentation_scope: String, + mark_projection: MarkProjection, timeout: Duration, transport: OtlpTransport, } @@ -109,6 +111,7 @@ impl Default for OpenTelemetryConfig { service_namespace: None, service_version: None, instrumentation_scope: "nemo-relay-otel".to_string(), + mark_projection: MarkProjection::default(), timeout: Duration::from_secs(3), transport: OtlpTransport::HttpBinary, } @@ -179,6 +182,12 @@ impl OpenTelemetryConfig { self.instrumentation_scope = scope.into(); self } + + /// Selects how point-in-time marks are represented in exported traces. + pub fn with_mark_projection(mut self, mark_projection: MarkProjection) -> Self { + self.mark_projection = mark_projection; + self + } } /// OpenTelemetry-backed NeMo Relay subscriber. @@ -204,6 +213,7 @@ impl OpenTelemetrySubscriber { Ok(Self::from_tracer_provider_with_scope( provider, config.instrumentation_scope, + config.mark_projection, )) } @@ -212,16 +222,35 @@ impl OpenTelemetrySubscriber { provider: SdkTracerProvider, instrumentation_scope: impl Into, ) -> Self { - Self::from_tracer_provider_with_scope(provider, instrumentation_scope.into()) + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + MarkProjection::default(), + ) + } + + /// Builds a subscriber from a tracer provider with an explicit mark projection. + pub fn from_tracer_provider_with_mark_projection( + provider: SdkTracerProvider, + instrumentation_scope: impl Into, + mark_projection: MarkProjection, + ) -> Self { + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + mark_projection, + ) } fn from_tracer_provider_with_scope( provider: SdkTracerProvider, instrumentation_scope: String, + mark_projection: MarkProjection, ) -> Self { - let processor = Arc::new(Mutex::new(OtelEventProcessor::new( + let processor = Arc::new(Mutex::new(OtelEventProcessor::new_with_mark_projection( provider, instrumentation_scope, + mark_projection, ))); let processor_for_callback = Arc::clone(&processor); let subscriber: EventSubscriberFn = Arc::new(move |event: &Event| { @@ -375,10 +404,20 @@ struct OtelEventProcessor { completed_span_order: VecDeque, provider: SdkTracerProvider, tracer: SdkTracer, + mark_projection: MarkProjection, } impl OtelEventProcessor { + #[cfg(test)] fn new(provider: SdkTracerProvider, instrumentation_scope: String) -> Self { + Self::new_with_mark_projection(provider, instrumentation_scope, MarkProjection::default()) + } + + fn new_with_mark_projection( + provider: SdkTracerProvider, + instrumentation_scope: String, + mark_projection: MarkProjection, + ) -> Self { let tracer = provider.tracer(instrumentation_scope); Self { active_spans: HashMap::new(), @@ -386,6 +425,7 @@ impl OtelEventProcessor { completed_span_order: VecDeque::new(), provider, tracer, + mark_projection, } } @@ -437,6 +477,10 @@ impl OtelEventProcessor { } fn process_mark(&mut self, event: &Event) { + if self.mark_projection == MarkProjection::Tool && !is_llm_chunk_mark(event) { + self.process_mark_as_tool(event); + return; + } let mark_name = event.name().to_string(); let timestamp = to_system_time(*event.timestamp()); let attributes = mark_attributes(event); @@ -460,6 +504,26 @@ impl OtelEventProcessor { span.end_with_timestamp(timestamp); } + fn process_mark_as_tool(&mut self, event: &Event) { + let timestamp = to_system_time(*event.timestamp()); + let orphan = self.find_parent_span(event).is_none(); + let mut attributes = mark_attributes(event); + attributes.push(KeyValue::new("nemo_relay.mark.projection", "tool")); + attributes.push(KeyValue::new("nemo_relay.scope_type", "tool")); + if orphan { + attributes.push(KeyValue::new("nemo_relay.mark.orphan", true)); + } + + let mut span = self + .tracer + .span_builder(format!("mark:{}", event.name())) + .with_kind(SpanKind::Internal) + .with_start_time(timestamp) + .start_with_context(&self.tracer, &self.parent_context(event)); + span.set_attributes(attributes); + span.end_with_timestamp(timestamp); + } + fn parent_context(&self, event: &Event) -> Context { if let Some(active_span) = self.find_parent_span(event) { return Context::new().with_remote_span_context(active_span.span_context.clone()); @@ -645,6 +709,17 @@ fn mark_attributes(event: &Event) -> Vec { "nemo_relay.mark.metadata_json", event.metadata(), ); + if let Some(category) = event.category() { + attributes.push(KeyValue::new( + "nemo_relay.mark.category", + category.as_str().to_string(), + )); + } + push_serialized( + &mut attributes, + "nemo_relay.mark.category_profile_json", + event.category_profile(), + ); attributes } diff --git a/crates/core/src/observability/plugin_component.rs b/crates/core/src/observability/plugin_component.rs index ff8e62197..87e4c6cdd 100644 --- a/crates/core/src/observability/plugin_component.rs +++ b/crates/core/src/observability/plugin_component.rs @@ -36,6 +36,7 @@ use crate::api::subscriber::{ scope_deregister_subscriber, try_scope_deregister_subscriber, try_scope_register_subscriber, }; use crate::error::FlowError; +use crate::observability::MarkProjection; use crate::observability::atif::{AtifAgentInfo, AtifExporter}; use crate::observability::atof::{ AtofEndpointConfig as CoreAtofEndpointConfig, AtofEndpointFieldNamePolicy, @@ -227,6 +228,10 @@ pub struct AtifSectionConfig { /// Default model name. #[serde(default = "default_model_name")] pub model_name: String, + /// Representation used for mark events: `event` or `tool`. + #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "mark_projection_schema"))] + pub mark_projection: MarkProjection, /// Tool definitions available to the agent. #[serde(default, skip_serializing_if = "Option::is_none")] pub tool_definitions: Option>, @@ -263,6 +268,7 @@ impl Default for AtifSectionConfig { agent_name: default_agent_name(), agent_version: default_agent_version(), model_name: default_model_name(), + mark_projection: MarkProjection::default(), tool_definitions: None, extra: None, output_directory: None, @@ -374,6 +380,10 @@ pub struct OtlpSectionConfig { /// Whether the subscriber is active. #[serde(default)] pub enabled: bool, + /// Representation used for mark events: `event` or `tool`. + #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "mark_projection_schema"))] + pub mark_projection: MarkProjection, /// OTLP transport: `http_binary` or `grpc`. #[serde(default = "default_otlp_transport")] #[cfg_attr(feature = "schema", schemars(schema_with = "otlp_transport_schema"))] @@ -408,6 +418,7 @@ impl Default for OtlpSectionConfig { fn default() -> Self { Self { enabled: false, + mark_projection: MarkProjection::default(), transport: default_otlp_transport(), endpoint: None, headers: HashMap::new(), @@ -476,6 +487,7 @@ crate::editor_config! { agent_name => { label: "agent_name", kind: String }, agent_version => { label: "agent_version", kind: String }, model_name => { label: "model_name", kind: String }, + mark_projection => { label: "mark_projection", kind: Enum, values: ["event", "tool"] }, tool_definitions => { label: "tool_definitions", kind: Json, optional: true }, extra => { label: "extra", kind: Json, optional: true }, output_directory => { label: "output_directory", kind: String, optional: true }, @@ -487,6 +499,7 @@ crate::editor_config! { crate::editor_config! { impl OtlpSectionConfig { enabled => { label: "enabled", kind: Boolean }, + mark_projection => { label: "mark_projection", kind: Enum, values: ["event", "tool"] }, transport => { label: "transport", kind: Enum, values: ["http_binary", "grpc"] }, endpoint => { label: "endpoint", kind: String, optional: true }, headers => { label: "headers", kind: StringMap }, @@ -581,6 +594,13 @@ fn otlp_transport_schema( string_enum_schema(generator, &["http_binary", "grpc"], Some("http_binary")) } +#[cfg(feature = "schema")] +fn mark_projection_schema( + generator: &mut schemars::r#gen::SchemaGenerator, +) -> schemars::schema::Schema { + string_enum_schema(generator, &["event", "tool"], Some("event")) +} + #[cfg(feature = "schema")] fn string_enum_schema( generator: &mut schemars::r#gen::SchemaGenerator, @@ -929,7 +949,8 @@ impl AtifDispatcher { // subscriber is attached after that start event has already been // emitted. let session_id = event.uuid().to_string(); - let exporter = AtifExporter::new(session_id.clone(), self.agent_info()); + let exporter = AtifExporter::new(session_id.clone(), self.agent_info()) + .with_mark_projection(self.config.mark_projection); (exporter.subscriber())(event); let (filename, local_path) = self.prepare_destination(&session_id); self.scope_owners.insert(event.uuid(), event.uuid()); @@ -1347,6 +1368,7 @@ fn build_otel_config(section: OtlpSectionConfig) -> PluginResult PluginResult bool { #[cfg(feature = "schema")] fn schema_property_has_enum(schema: &Json, name: &str, expected: &[&str]) -> bool { - schema_property(schema, name) - .and_then(|property| property.get("enum")) - .and_then(Json::as_array) - .is_some_and(|values| { - expected - .iter() - .all(|expected| values.iter().any(|value| value == *expected)) - }) + schema_property_matches(schema, name, &|property| { + property + .get("enum") + .and_then(Json::as_array) + .is_some_and(|values| { + expected + .iter() + .all(|expected| values.iter().any(|value| value == *expected)) + }) + }) } #[cfg(feature = "schema")] fn schema_property_has_default(schema: &Json, name: &str, expected: Json) -> bool { - schema_property(schema, name) - .and_then(|property| property.get("default")) - .is_some_and(|default| default == &expected) + schema_property_matches(schema, name, &|property| { + property + .get("default") + .is_some_and(|default| default == &expected) + }) +} + +#[cfg(feature = "schema")] +fn schema_property_matches(schema: &Json, name: &str, predicate: &impl Fn(&Json) -> bool) -> bool { + match schema { + Json::Object(object) => { + if object + .get("properties") + .and_then(Json::as_object) + .and_then(|properties| properties.get(name)) + .is_some_and(predicate) + { + return true; + } + object + .values() + .any(|value| schema_property_matches(value, name, predicate)) + } + Json::Array(values) => values + .iter() + .any(|value| schema_property_matches(value, name, predicate)), + _ => false, + } } #[cfg(feature = "schema")] @@ -189,10 +216,12 @@ fn default_config_and_component_conversion_cover_public_shape() { assert_eq!(atif.agent_name, "NeMo Relay"); assert_eq!(atif.agent_version, env!("CARGO_PKG_VERSION")); assert_eq!(atif.model_name, "unknown"); + assert_eq!(atif.mark_projection, MarkProjection::Event); assert_eq!(atif.filename_template, "nemo-relay-atif-{session_id}.json"); let otlp = OtlpSectionConfig::default(); assert!(!otlp.enabled); + assert_eq!(otlp.mark_projection, MarkProjection::Event); assert_eq!(otlp.transport, "http_binary"); assert_eq!(otlp.service_name, "nemo-relay"); assert_eq!(otlp.timeout_millis, 3_000); @@ -211,6 +240,34 @@ fn default_config_and_component_conversion_cover_public_shape() { assert_eq!(generic.config["atif"]["agent_name"], json!("NeMo Relay")); } +#[test] +fn mark_projection_parses_per_exporter_and_rejects_unknown_values() { + let atif: AtifSectionConfig = serde_json::from_value(json!({ + "mark_projection": "tool" + })) + .unwrap(); + let otlp: OtlpSectionConfig = serde_json::from_value(json!({ + "mark_projection": "tool" + })) + .unwrap(); + assert_eq!(atif.mark_projection, MarkProjection::Tool); + assert_eq!(otlp.mark_projection, MarkProjection::Tool); + + let error = serde_json::from_value::(json!({ + "mark_projection": "span" + })) + .unwrap_err(); + assert!(error.to_string().contains("unknown variant `span`")); + + let report = validate_plugin_config(&plugin_config(json!({ + "opentelemetry": {"mark_projection": "span"} + }))); + assert!(report.has_errors()); + assert!(report.diagnostics.iter().any(|diagnostic| diagnostic.code + == "observability.invalid_plugin_config" + && diagnostic.message.contains("unknown variant `span`"))); +} + #[cfg(feature = "schema")] #[test] fn schema_contains_every_supported_observability_option() { @@ -230,6 +287,7 @@ fn schema_contains_every_supported_observability_option() { "agent_name", "agent_version", "model_name", + "mark_projection", "tool_definitions", "extra", "filename_template", @@ -262,6 +320,11 @@ fn schema_contains_every_supported_observability_option() { "transport", &["http_binary", "grpc"] )); + assert!(schema_property_has_enum( + &schema, + "mark_projection", + &["event", "tool"] + )); assert!(schema_property_has_default( &schema, "mode", diff --git a/crates/node/observability.d.ts b/crates/node/observability.d.ts index f12ede6ec..1d0201a00 100644 --- a/crates/node/observability.d.ts +++ b/crates/node/observability.d.ts @@ -47,6 +47,7 @@ export interface AtifConfig { agent_name?: string; agent_version?: string; model_name?: string; + mark_projection?: 'event' | 'tool'; tool_definitions?: Record[]; extra?: Record; output_directory?: string; @@ -56,6 +57,7 @@ export interface AtifConfig { export interface OtlpConfig { enabled?: boolean; + mark_projection?: 'event' | 'tool'; transport?: 'http_binary' | 'grpc' | string; endpoint?: string; headers?: Record; diff --git a/crates/node/observability.js b/crates/node/observability.js index 7ce69c935..637fa492b 100644 --- a/crates/node/observability.js +++ b/crates/node/observability.js @@ -43,6 +43,7 @@ function atifConfig(config = {}) { enabled: false, agent_name: 'NeMo Relay', model_name: 'unknown', + mark_projection: 'event', filename_template: 'nemo-relay-atif-{session_id}.json', ...config, }; @@ -57,6 +58,7 @@ function atifConfig(config = {}) { function otlpConfig(config = {}) { return { enabled: false, + mark_projection: 'event', transport: 'http_binary', headers: {}, resource_attributes: {}, diff --git a/crates/node/tests/observability_plugin_tests.mjs b/crates/node/tests/observability_plugin_tests.mjs index e0741d079..a5d91f2eb 100644 --- a/crates/node/tests/observability_plugin_tests.mjs +++ b/crates/node/tests/observability_plugin_tests.mjs @@ -25,16 +25,20 @@ describe('observability plugin helpers', () => { enabled: false, agent_name: 'NeMo Relay', model_name: 'unknown', + mark_projection: 'event', filename_template: 'nemo-relay-atif-{session_id}.json', }); assert.deepEqual(observability.otlpConfig(), { enabled: false, + mark_projection: 'event', transport: 'http_binary', headers: {}, resource_attributes: {}, service_name: 'nemo-relay', timeout_millis: 3000, }); + assert.equal(observability.atifConfig({ mark_projection: 'tool' }).mark_projection, 'tool'); + assert.equal(observability.otlpConfig({ mark_projection: 'tool' }).mark_projection, 'tool'); const component = observability.ComponentSpec({ version: 1, atof: observability.atofConfig() }); assert.equal(component.kind, observability.OBSERVABILITY_PLUGIN_KIND); diff --git a/docs/configure-plugins/observability/atif.mdx b/docs/configure-plugins/observability/atif.mdx index c3e16c9ff..7d956e1ae 100644 --- a/docs/configure-plugins/observability/atif.mdx +++ b/docs/configure-plugins/observability/atif.mdx @@ -53,6 +53,7 @@ The following table describes the top-level ATIF settings: | `agent_name` | `NeMo Relay` | Agent metadata written into the trajectory. | | `agent_version` | NeMo Relay crate version | Agent version metadata. | | `model_name` | `unknown` | Default model metadata when no call-level model is present. | +| `mark_projection` | `event` | `event` preserves marks as system steps. `tool` emits deterministic agent/tool steps for trajectory viewers that render tool calls but not system events. | | `tool_definitions` | Omitted | Optional ATIF tool metadata. | | `extra` | Omitted | Optional ATIF agent metadata. | | `output_directory` | Current working directory | Directory containing trajectory files. Ignored when `storage` is non-empty. | @@ -194,6 +195,12 @@ per-destination sink failures to the failed sink. ## Expected Output +By default, marks remain point-in-time `system` steps with their payload, +category, category profile, timestamp, and ancestry preserved in `extra`. Set +`mark_projection = "tool"` when the target viewer requires a visible tool-call +projection. The projected step uses `llm_call_count = 0`, retains the original +mark data in `extra`, and does not change the canonical ATOF event. + The exporter translates NeMo Relay lifecycle events into ATIF v1.7 trajectory data. LLM start and end events become model steps, tool events become tool calls and observations, and scope nesting contributes lineage metadata. diff --git a/docs/configure-plugins/observability/openinference.mdx b/docs/configure-plugins/observability/openinference.mdx index 56f3e8edc..e0542be8c 100644 --- a/docs/configure-plugins/observability/openinference.mdx +++ b/docs/configure-plugins/observability/openinference.mdx @@ -61,6 +61,7 @@ OpenInference uses the same OTLP section shape as | Field | Default | Notes | |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | +| `mark_projection` | `event` | `event` attaches marks to active parent spans. `tool` emits zero-duration OpenInference `TOOL` child spans. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | | `headers` | `{}` | String-to-string exporter headers. | @@ -83,6 +84,12 @@ and invocation parameters are emitted when typed codec annotations supply them. Exported LLM attributes exclude request headers and other non-observable transport metadata. +The default `event` projection preserves marks as span events. With +`mark_projection = "tool"`, each non-streaming mark becomes a zero-duration child span with +`openinference.span.kind = "TOOL"` and the original mark payload, category, +profile, timestamp, and parent identifiers retained as attributes. High-volume +`llm.chunk` marks remain span events. + Each lifecycle span includes `nemo_relay.uuid` and `nemo_relay.parent_uuid` attributes. These values match ATIF `step.extra.ancestry.function_id` and `step.extra.ancestry.parent_id` for the same events. For plugin-managed ATIF, diff --git a/docs/configure-plugins/observability/opentelemetry.mdx b/docs/configure-plugins/observability/opentelemetry.mdx index 70606a24d..38eb6c97a 100644 --- a/docs/configure-plugins/observability/opentelemetry.mdx +++ b/docs/configure-plugins/observability/opentelemetry.mdx @@ -55,6 +55,7 @@ The following table describes OpenTelemetry exporter settings: | Field | Default | Notes | |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | +| `mark_projection` | `event` | `event` attaches marks to active parent spans. `tool` emits zero-duration child spans for trace-tree visibility. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | | `headers` | `{}` | String-to-string exporter headers. | @@ -71,6 +72,13 @@ The collector should receive OTLP trace export requests. The tracing backend should show spans for NeMo Relay scopes, tools, LLM calls, and marks grouped by root scope. +The default `event` projection follows the OpenTelemetry span-event model. Set +`mark_projection = "tool"` only when the target backend needs non-streaming +marks represented as visible child spans. Projected marks use `SpanKind::Internal`, carry +`nemo_relay.mark.projection = "tool"`, and retain mark UUID, parentage, +category/profile, and payload attributes. High-volume `llm.chunk` marks remain +span events. + Each lifecycle span includes `nemo_relay.uuid` and `nemo_relay.parent_uuid` attributes. These values match ATIF `step.extra.ancestry.function_id` and `step.extra.ancestry.parent_id` for the same events. For plugin-managed ATIF, diff --git a/go/nemo_relay/observability_plugin.go b/go/nemo_relay/observability_plugin.go index 3660dfe4b..0bb411fed 100644 --- a/go/nemo_relay/observability_plugin.go +++ b/go/nemo_relay/observability_plugin.go @@ -8,6 +8,16 @@ import "encoding/json" // ObservabilityPluginKind is the top-level plugin kind used by the core observability component. const ObservabilityPluginKind = "observability" +// ObservabilityMarkProjection controls how point-in-time marks are exported. +type ObservabilityMarkProjection string + +const ( + // ObservabilityMarkProjectionEvent preserves marks as events. + ObservabilityMarkProjectionEvent ObservabilityMarkProjection = "event" + // ObservabilityMarkProjectionTool emits visible tool projections. + ObservabilityMarkProjectionTool ObservabilityMarkProjection = "tool" +) + // ObservabilityConfig is the canonical Go shape for the observability plugin config document. type ObservabilityConfig struct { Version uint32 `json:"version,omitempty"` @@ -42,6 +52,7 @@ type ObservabilityAtifConfig struct { AgentName string `json:"agent_name,omitempty"` AgentVersion string `json:"agent_version,omitempty"` ModelName string `json:"model_name,omitempty"` + MarkProjection ObservabilityMarkProjection `json:"mark_projection,omitempty"` ToolDefinitions []map[string]any `json:"tool_definitions,omitempty"` Extra map[string]any `json:"extra,omitempty"` OutputDirectory string `json:"output_directory,omitempty"` @@ -128,16 +139,17 @@ func (config ObservabilityHttpStorageConfig) MarshalJSON() ([]byte, error) { // ObservabilityOtlpConfig configures OpenTelemetry or OpenInference OTLP export. type ObservabilityOtlpConfig struct { - Enabled bool `json:"enabled,omitempty"` - Transport string `json:"transport,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - Headers map[string]string `json:"headers,omitempty"` - ResourceAttributes map[string]string `json:"resource_attributes,omitempty"` - ServiceName string `json:"service_name,omitempty"` - ServiceNamespace string `json:"service_namespace,omitempty"` - ServiceVersion string `json:"service_version,omitempty"` - InstrumentationScope string `json:"instrumentation_scope,omitempty"` - TimeoutMillis uint64 `json:"timeout_millis,omitempty"` + Enabled bool `json:"enabled,omitempty"` + MarkProjection ObservabilityMarkProjection `json:"mark_projection,omitempty"` + Transport string `json:"transport,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + ResourceAttributes map[string]string `json:"resource_attributes,omitempty"` + ServiceName string `json:"service_name,omitempty"` + ServiceNamespace string `json:"service_namespace,omitempty"` + ServiceVersion string `json:"service_version,omitempty"` + InstrumentationScope string `json:"instrumentation_scope,omitempty"` + TimeoutMillis uint64 `json:"timeout_millis,omitempty"` } // ObservabilityComponentSpec wraps one observability config as a top-level plugin component. @@ -163,6 +175,7 @@ func NewObservabilityAtifConfig() ObservabilityAtifConfig { return ObservabilityAtifConfig{ AgentName: "NeMo Relay", ModelName: "unknown", + MarkProjection: ObservabilityMarkProjectionEvent, FilenameTemplate: "nemo-relay-atif-{session_id}.json", } } @@ -181,6 +194,7 @@ func NewObservabilityHttpStorageConfig(endpoint string) ObservabilityHttpStorage func NewObservabilityOtlpConfig() ObservabilityOtlpConfig { return ObservabilityOtlpConfig{ Transport: "http_binary", + MarkProjection: ObservabilityMarkProjectionEvent, Headers: map[string]string{}, ResourceAttributes: map[string]string{}, ServiceName: "nemo-relay", diff --git a/go/nemo_relay/observability_plugin_test.go b/go/nemo_relay/observability_plugin_test.go index 0ad8e306f..b3d28901e 100644 --- a/go/nemo_relay/observability_plugin_test.go +++ b/go/nemo_relay/observability_plugin_test.go @@ -43,7 +43,7 @@ func TestObservabilityConfigHelpers(t *testing.T) { FieldNamePolicy: "replace_dots", }} atif := NewObservabilityAtifConfig() - if atif.Enabled || atif.AgentName != "NeMo Relay" || atif.ModelName != "unknown" || atif.FilenameTemplate != "nemo-relay-atif-{session_id}.json" { + if atif.Enabled || atif.AgentName != "NeMo Relay" || atif.ModelName != "unknown" || atif.MarkProjection != ObservabilityMarkProjectionEvent || atif.FilenameTemplate != "nemo-relay-atif-{session_id}.json" { t.Fatalf("unexpected ATIF defaults: %#v", atif) } allowHTTP := false @@ -64,12 +64,15 @@ func TestObservabilityConfigHelpers(t *testing.T) { httpStorage, } otlp := NewObservabilityOtlpConfig() - if otlp.Enabled || otlp.Transport != "http_binary" || otlp.ServiceName != "nemo-relay" || otlp.TimeoutMillis != 3000 { + if otlp.Enabled || otlp.MarkProjection != ObservabilityMarkProjectionEvent || otlp.Transport != "http_binary" || otlp.ServiceName != "nemo-relay" || otlp.TimeoutMillis != 3000 { t.Fatalf("unexpected OTLP defaults: %#v", otlp) } + atif.MarkProjection = ObservabilityMarkProjectionTool + otlp.MarkProjection = ObservabilityMarkProjectionTool config.Atof = &atof config.Atif = &atif + config.OpenTelemetry = &otlp wrapped := ObservabilityComponent(config) if wrapped.Kind != ObservabilityPluginKind || !wrapped.Enabled { t.Fatalf("unexpected component wrapper: %#v", wrapped) @@ -94,6 +97,9 @@ func TestObservabilityConfigHelpers(t *testing.T) { t.Fatalf("expected field_name_policy in serialized component, got %s", serialized) } assertWrappedAtifStorageConfig(t, wrapped.Config["atif"].(map[string]any)) + if wrapped.Config["atif"].(map[string]any)["mark_projection"] != "tool" || wrapped.Config["opentelemetry"].(map[string]any)["mark_projection"] != "tool" { + t.Fatalf("expected tool mark projection in serialized config: %#v", wrapped.Config) + } } func assertS3StorageConfig(t *testing.T, storage ObservabilityS3StorageConfig) { diff --git a/python/nemo_relay/observability.py b/python/nemo_relay/observability.py index 38d90e9c0..f57edb34b 100644 --- a/python/nemo_relay/observability.py +++ b/python/nemo_relay/observability.py @@ -10,6 +10,8 @@ from nemo_relay import Json, JsonObject, UnsupportedBehavior +MarkProjection = Literal["event", "tool"] + class _SupportsToDict(Protocol): def to_dict(self) -> JsonObject: ... @@ -165,6 +167,7 @@ class AtifConfig: agent_name: str = "NeMo Relay" agent_version: str | None = None model_name: str = "unknown" + mark_projection: MarkProjection = "event" tool_definitions: list[JsonObject] | None = None extra: JsonObject | None = None output_directory: str | None = None @@ -178,6 +181,7 @@ def to_dict(self) -> JsonObject: "agent_name": self.agent_name, "agent_version": self.agent_version, "model_name": self.model_name, + "mark_projection": self.mark_projection, "tool_definitions": self.tool_definitions, "extra": self.extra, "output_directory": self.output_directory, @@ -194,6 +198,7 @@ class OtlpConfig: """Shared OpenTelemetry/OpenInference OTLP export settings.""" enabled: bool = False + mark_projection: MarkProjection = "event" transport: Literal["http_binary", "grpc"] = "http_binary" endpoint: str | None = None headers: dict[str, str] = field(default_factory=dict) @@ -209,6 +214,7 @@ def to_dict(self) -> JsonObject: return _normalize_object( { "enabled": self.enabled, + "mark_projection": self.mark_projection, "transport": self.transport, "endpoint": self.endpoint, "headers": self.headers, @@ -272,6 +278,7 @@ def to_dict(self) -> JsonObject: "AtofConfig", "AtifConfig", "HttpStorageConfig", + "MarkProjection", "S3StorageConfig", "OtlpConfig", "ObservabilityConfig", diff --git a/python/nemo_relay/observability.pyi b/python/nemo_relay/observability.pyi index 11e5b4d60..a117e0d52 100644 --- a/python/nemo_relay/observability.pyi +++ b/python/nemo_relay/observability.pyi @@ -10,6 +10,8 @@ from typing import Literal from nemo_relay import JsonObject, UnsupportedBehavior +MarkProjection = Literal["event", "tool"] + @dataclass(slots=True) class ConfigPolicy: unknown_component: UnsupportedBehavior = ... @@ -61,6 +63,7 @@ class AtifConfig: agent_name: str = ... agent_version: str | None = ... model_name: str = ... + mark_projection: MarkProjection = ... tool_definitions: list[JsonObject] | None = ... extra: JsonObject | None = ... output_directory: str | None = ... @@ -71,6 +74,7 @@ class AtifConfig: @dataclass(slots=True) class OtlpConfig: enabled: bool = ... + mark_projection: MarkProjection = ... transport: Literal["http_binary", "grpc"] = ... endpoint: str | None = ... headers: dict[str, str] = field(default_factory=dict) diff --git a/python/tests/test_observability_plugin.py b/python/tests/test_observability_plugin.py index 7dd34841b..2befca774 100644 --- a/python/tests/test_observability_plugin.py +++ b/python/tests/test_observability_plugin.py @@ -34,16 +34,20 @@ def test_defaults_and_component_wrapper(self): "enabled": False, "agent_name": "NeMo Relay", "model_name": "unknown", + "mark_projection": "event", "filename_template": "nemo-relay-atif-{session_id}.json", } assert OtlpConfig().to_dict() == { "enabled": False, + "mark_projection": "event", "transport": "http_binary", "headers": {}, "resource_attributes": {}, "service_name": "nemo-relay", "timeout_millis": 3000, } + assert AtifConfig(mark_projection="tool").to_dict()["mark_projection"] == "tool" + assert OtlpConfig(mark_projection="tool").to_dict()["mark_projection"] == "tool" wrapped = ComponentSpec(ObservabilityConfig(atof=AtofConfig())).to_dict() assert wrapped["kind"] == OBSERVABILITY_PLUGIN_KIND From b5b11ed08727aa167cce974d841a7f53279123f5 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 7 Jul 2026 15:12:45 -0600 Subject: [PATCH 02/14] fix(observability): project data-less marks Signed-off-by: Bryan Bednarski --- crates/core/src/observability/atif.rs | 28 ++- .../tests/integration/middleware_tests.rs | 185 ++++++++++++++++++ crates/core/tests/unit/atif_tests.rs | 66 +++++++ .../unit/observability/openinference_tests.rs | 24 +++ 4 files changed, 294 insertions(+), 9 deletions(-) diff --git a/crates/core/src/observability/atif.rs b/crates/core/src/observability/atif.rs index 8dd78820f..e5c3bb571 100644 --- a/crates/core/src/observability/atif.rs +++ b/crates/core/src/observability/atif.rs @@ -2620,16 +2620,16 @@ impl StepConversionState { return; } self.flush_observations(); + if self.mark_projection == MarkProjection::Tool { + self.handle_mark_as_tool(mark, lookups, mark.data()); + return; + } let Some(data) = mark.data() else { return; }; if is_empty_mark_payload(data) { return; } - if self.mark_projection == MarkProjection::Tool { - self.handle_mark_as_tool(mark, lookups, data); - return; - } let extra = AtifStepExtra { ancestry: build_ancestry(mark, &lookups.name_map), invocation: Some(AtifInvocationInfo { @@ -2668,7 +2668,12 @@ impl StepConversionState { }); } - fn handle_mark_as_tool(&mut self, mark: &Event, lookups: &EventLookupMaps, data: &Json) { + fn handle_mark_as_tool( + &mut self, + mark: &Event, + lookups: &EventLookupMaps, + data: Option<&Json>, + ) { self.finalize_agent_extra(); let ancestry = build_ancestry(mark, &lookups.name_map); @@ -2681,7 +2686,7 @@ impl StepConversionState { }; let source_call_id = format!("mark:{}", mark.uuid()); let mut observation_extra = event_extra(mark); - if let Json::Object(extra) = &mut observation_extra { + if let (Some(data), Json::Object(extra)) = (data, &mut observation_extra) { extra.insert("event_payload".to_string(), data.clone()); } let extra = AtifStepExtra { @@ -2689,7 +2694,7 @@ impl StepConversionState { invocation: Some(invocation.clone()), llm_request: None, llm_response: None, - event_payload: Some(data.clone()), + event_payload: data.cloned(), event_category: mark .category() .map(|category| category.as_str().to_string()), @@ -2711,13 +2716,18 @@ impl StepConversionState { tool_calls: Some(vec![AtifToolCall { tool_call_id: source_call_id.clone(), function_name: mark.name().to_string(), - arguments: data.clone(), + // ATIF requires tool-call arguments to be a JSON object. Keep + // the original payload absence in `extra.event_payload` while + // using an empty object for the schema-required arguments. + arguments: data + .cloned() + .unwrap_or_else(|| Json::Object(serde_json::Map::new())), extra: Some(event_extra(mark)), }]), observation: Some(AtifObservation { results: vec![AtifObservationResult { source_call_id: Some(source_call_id), - content: Some(mark_message(mark, data)), + content: Some(mark_message(mark, data.unwrap_or(&Json::Null))), subagent_trajectory_ref: None, extra: Some(observation_extra), }], diff --git a/crates/core/tests/integration/middleware_tests.rs b/crates/core/tests/integration/middleware_tests.rs index b978195bf..866dbb447 100644 --- a/crates/core/tests/integration/middleware_tests.rs +++ b/crates/core/tests/integration/middleware_tests.rs @@ -55,7 +55,17 @@ use nemo_relay::api::tool::{ tool_conditional_execution, tool_request_intercepts, }; use nemo_relay::error::FlowError; +#[cfg(all(feature = "otel", feature = "openinference"))] +use nemo_relay::observability::MarkProjection; +#[cfg(all(feature = "otel", feature = "openinference"))] +use nemo_relay::observability::atif::{AtifAgentInfo, AtifExporter}; +#[cfg(all(feature = "otel", feature = "openinference"))] +use nemo_relay::observability::openinference::OpenInferenceSubscriber; +#[cfg(all(feature = "otel", feature = "openinference"))] +use nemo_relay::observability::otel::OpenTelemetrySubscriber; use nemo_relay::plugin::{PluginRegistrationContext, rollback_registrations}; +#[cfg(all(feature = "otel", feature = "openinference"))] +use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; use serde_json::json; // All tests share the global context, so we serialize them. @@ -789,6 +799,181 @@ async fn test_tool_execution_outcome_marks_follow_end_with_tool_parentage() { deregister_subscriber("tool_outcome_mark_observer").unwrap(); } +#[cfg(all(feature = "otel", feature = "openinference"))] +#[tokio::test] +async fn test_managed_tool_pending_marks_project_through_all_exporters() { + let _lock = TEST_MUTEX.lock().unwrap(); + reset_global(); + setup_isolated_thread(); + + let events = Arc::new(Mutex::new(Vec::::new())); + let captured = Arc::clone(&events); + register_subscriber( + "managed_tool_projection_events", + Arc::new(move |event| captured.lock().unwrap().push(event.clone())), + ) + .unwrap(); + + let atif = AtifExporter::new( + "managed-tool-projection".to_string(), + AtifAgentInfo { + name: "test-agent".to_string(), + version: "1.0.0".to_string(), + model_name: None, + tool_definitions: None, + extra: None, + }, + ) + .with_mark_projection(MarkProjection::Tool); + register_subscriber("managed_tool_projection_atif", atif.subscriber()).unwrap(); + + let otel_exporter = InMemorySpanExporterBuilder::new().build(); + let otel_provider = SdkTracerProvider::builder() + .with_simple_exporter(otel_exporter.clone()) + .build(); + let otel = OpenTelemetrySubscriber::from_tracer_provider_with_mark_projection( + otel_provider, + "managed-tool-projection-otel", + MarkProjection::Tool, + ); + register_subscriber("managed_tool_projection_otel", otel.subscriber()).unwrap(); + + let openinference_exporter = InMemorySpanExporterBuilder::new().build(); + let openinference_provider = SdkTracerProvider::builder() + .with_simple_exporter(openinference_exporter.clone()) + .build(); + let openinference = OpenInferenceSubscriber::from_tracer_provider_with_mark_projection( + openinference_provider, + "managed-tool-projection-openinference", + MarkProjection::Tool, + ); + register_subscriber( + "managed_tool_projection_openinference", + openinference.subscriber(), + ) + .unwrap(); + + register_tool_execution_intercept( + "managed_tool_projection_intercept", + 1, + Arc::new(|_name, args, next| { + Box::pin(async move { + let result = next(args).await?; + Ok( + ToolExecutionInterceptOutcome::new(result).with_pending_mark( + PendingMarkSpec::builder() + .name("plugin.output_compacted") + .category(EventCategory::custom()) + .category_profile( + CategoryProfile::builder() + .subtype("example.compaction") + .build(), + ) + .data(json!({"saved_tokens": 12})) + .metadata(json!({"source": "test"})) + .build(), + ), + ) + }) + }), + ) + .unwrap(); + + let result = tool_call_execute( + nemo_relay::api::tool::ToolCallExecuteParams::builder() + .name("managed-tool") + .args(json!({"value": 42})) + .func(Arc::new(|args| Box::pin(async move { Ok(args) }))) + .build(), + ) + .await + .unwrap(); + assert_eq!(result, json!({"value": 42})); + + flush_subscribers().unwrap(); + let captured = events.lock().unwrap(); + let tool_end_index = captured + .iter() + .position(|event| { + event.name() == "managed-tool" && event.scope_category() == Some(ScopeCategory::End) + }) + .unwrap(); + let mark_index = captured + .iter() + .position(|event| event.name() == "plugin.output_compacted") + .unwrap(); + assert!(tool_end_index < mark_index); + let tool_end = &captured[tool_end_index]; + let mark = &captured[mark_index]; + assert_eq!(mark.parent_uuid(), Some(tool_end.uuid())); + assert!(mark.timestamp() > tool_end.timestamp()); + drop(captured); + + let trajectory = atif.export().unwrap(); + let mark_step = trajectory + .steps + .iter() + .find(|step| { + step.tool_calls.as_deref().is_some_and(|calls| { + calls + .iter() + .any(|call| call.function_name == "plugin.output_compacted") + }) + }) + .unwrap(); + assert_eq!(mark_step.llm_call_count, Some(0)); + let projected_call = &mark_step.tool_calls.as_ref().unwrap()[0]; + assert_eq!(projected_call.arguments, json!({"saved_tokens": 12})); + assert_eq!( + mark_step.observation.as_ref().unwrap().results[0] + .source_call_id + .as_deref(), + Some(projected_call.tool_call_id.as_str()) + ); + + otel.force_flush().unwrap(); + let otel_spans = otel_exporter.get_finished_spans().unwrap(); + let otel_tool = otel_spans + .iter() + .find(|span| span.name.as_ref() == "managed-tool") + .unwrap(); + let otel_mark = otel_spans + .iter() + .find(|span| span.name.as_ref() == "mark:plugin.output_compacted") + .unwrap(); + assert_eq!(otel_mark.parent_span_id, otel_tool.span_context.span_id()); + assert!(otel_mark.start_time > otel_tool.end_time); + assert!(otel_mark.attributes.iter().any(|attribute| { + attribute.key.as_str() == "nemo_relay.mark.projection" + && attribute.value.to_string() == "tool" + })); + + openinference.force_flush().unwrap(); + let openinference_spans = openinference_exporter.get_finished_spans().unwrap(); + let openinference_tool = openinference_spans + .iter() + .find(|span| span.name.as_ref() == "managed-tool") + .unwrap(); + let openinference_mark = openinference_spans + .iter() + .find(|span| span.name.as_ref() == "mark:plugin.output_compacted") + .unwrap(); + assert_eq!( + openinference_mark.parent_span_id, + openinference_tool.span_context.span_id() + ); + assert!(openinference_mark.start_time > openinference_tool.end_time); + assert!(openinference_mark.attributes.iter().any(|attribute| { + attribute.key.as_str() == "openinference.span.kind" && attribute.value.to_string() == "TOOL" + })); + + deregister_tool_execution_intercept("managed_tool_projection_intercept").unwrap(); + deregister_subscriber("managed_tool_projection_events").unwrap(); + deregister_subscriber("managed_tool_projection_atif").unwrap(); + deregister_subscriber("managed_tool_projection_otel").unwrap(); + deregister_subscriber("managed_tool_projection_openinference").unwrap(); +} + #[tokio::test] async fn test_tool_execution_error_discards_downstream_pending_marks() { let _lock = TEST_MUTEX.lock().unwrap(); diff --git a/crates/core/tests/unit/atif_tests.rs b/crates/core/tests/unit/atif_tests.rs index e7cdbdfe9..96acdfef0 100644 --- a/crates/core/tests/unit/atif_tests.rs +++ b/crates/core/tests/unit/atif_tests.rs @@ -1935,6 +1935,72 @@ fn test_exporter_tool_projection_renders_generic_mark_as_deterministic_tool_step ); } +#[test] +fn test_exporter_tool_projection_keeps_data_less_and_empty_marks_visible() { + let root_uuid = Uuid::now_v7(); + let exporter = AtifExporter::new(root_uuid.to_string(), make_agent_info()) + .with_mark_projection(MarkProjection::Tool); + let base = base_timestamp(); + + let mut root_start = event_builder(root_uuid, EventType::Start) + .name("neutral-agent") + .scope_type(ScopeType::Agent) + .build(); + let mut data_less_mark = Event::Mark(MarkEvent::new( + BaseEvent::builder() + .parent_uuid(root_uuid) + .name("plugin.checkpoint") + .build(), + None, + None, + )); + let mut empty_mark = Event::Mark(MarkEvent::new( + BaseEvent::builder() + .parent_uuid(root_uuid) + .name("plugin.empty_checkpoint") + .data(json!({})) + .build(), + None, + None, + )); + let mut root_end = event_builder(root_uuid, EventType::End) + .name("neutral-agent") + .scope_type(ScopeType::Agent) + .build(); + set_event_timestamp(&mut root_start, base); + set_event_timestamp( + &mut data_less_mark, + base + chrono::Duration::milliseconds(1), + ); + set_event_timestamp(&mut empty_mark, base + chrono::Duration::milliseconds(2)); + set_event_timestamp(&mut root_end, base + chrono::Duration::milliseconds(3)); + + exporter.state.lock().unwrap().events.extend([ + root_start, + data_less_mark, + empty_mark, + root_end, + ]); + + let trajectory = exporter.export().unwrap(); + assert_atif_v17_shape(&trajectory); + assert_eq!(trajectory.steps.len(), 2); + + let data_less_tool = &trajectory.steps[0].tool_calls.as_ref().unwrap()[0]; + assert_eq!(data_less_tool.function_name, "plugin.checkpoint"); + assert_eq!(data_less_tool.arguments, json!({})); + let data_less_extra: AtifStepExtra = + serde_json::from_value(trajectory.steps[0].extra.clone().unwrap()).unwrap(); + assert_eq!(data_less_extra.event_payload, None); + + let empty_tool = &trajectory.steps[1].tool_calls.as_ref().unwrap()[0]; + assert_eq!(empty_tool.function_name, "plugin.empty_checkpoint"); + assert_eq!(empty_tool.arguments, json!({})); + let empty_extra: AtifStepExtra = + serde_json::from_value(trajectory.steps[1].extra.clone().unwrap()).unwrap(); + assert_eq!(empty_extra.event_payload, Some(json!({}))); +} + #[test] fn test_exporter_openclaw_hook_only_fallbacks_preserve_stripped_content_and_explicit_metrics() { let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); diff --git a/crates/core/tests/unit/observability/openinference_tests.rs b/crates/core/tests/unit/observability/openinference_tests.rs index aa9655cbc..0799259cd 100644 --- a/crates/core/tests/unit/observability/openinference_tests.rs +++ b/crates/core/tests/unit/observability/openinference_tests.rs @@ -2225,6 +2225,30 @@ fn tool_projection_reuses_completed_parent_context_for_late_marks() { ); } +#[test] +fn tool_projection_keeps_llm_chunks_as_parent_span_events() { + let (provider, exporter) = make_provider(); + let mut processor = OpenInferenceEventProcessor::new_with_mark_projection( + provider.clone(), + "test-scope".to_string(), + MarkProjection::Tool, + ); + let parent_uuid = Uuid::now_v7(); + let start = make_start_event(parent_uuid, None, "llm", ScopeType::Llm, None); + let chunk = make_mark_event(Some(parent_uuid), "llm.chunk", Some(json!({"delta": "x"}))); + let end = make_end_event(parent_uuid, None, "llm", ScopeType::Llm, None); + + processor.process(&start); + processor.process(&chunk); + processor.process(&end); + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 1); + assert_eq!(spans[0].events.events.len(), 1); + assert_eq!(spans[0].events.events[0].name.as_ref(), "llm.chunk"); +} + #[test] fn late_parented_marks_reuse_completed_parent_trace_context() { let (provider, exporter) = make_provider(); From 76b5a6a7a661c53272908119bb6431a5da53a3a2 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 7 Jul 2026 15:52:25 -0600 Subject: [PATCH 03/14] fix(atif): normalize projected mark arguments Signed-off-by: Bryan Bednarski --- crates/core/src/observability/atif.rs | 19 ++++++++++++++---- crates/core/tests/unit/atif_tests.rs | 28 +++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/crates/core/src/observability/atif.rs b/crates/core/src/observability/atif.rs index e5c3bb571..423ce3468 100644 --- a/crates/core/src/observability/atif.rs +++ b/crates/core/src/observability/atif.rs @@ -2685,6 +2685,7 @@ impl StepConversionState { framework: Some("nemo_relay".to_string()), }; let source_call_id = format!("mark:{}", mark.uuid()); + let arguments = normalize_mark_tool_arguments(data); let mut observation_extra = event_extra(mark); if let (Some(data), Json::Object(extra)) = (data, &mut observation_extra) { extra.insert("event_payload".to_string(), data.clone()); @@ -2718,10 +2719,8 @@ impl StepConversionState { function_name: mark.name().to_string(), // ATIF requires tool-call arguments to be a JSON object. Keep // the original payload absence in `extra.event_payload` while - // using an empty object for the schema-required arguments. - arguments: data - .cloned() - .unwrap_or_else(|| Json::Object(serde_json::Map::new())), + // using a schema-valid object for the projected arguments. + arguments, extra: Some(event_extra(mark)), }]), observation: Some(AtifObservation { @@ -3498,6 +3497,18 @@ fn is_empty_mark_payload(data: &Json) -> bool { data.is_null() || data.as_object().is_some_and(|object| object.is_empty()) } +fn normalize_mark_tool_arguments(data: Option<&Json>) -> Json { + match data { + None | Some(Json::Null) => Json::Object(serde_json::Map::new()), + Some(Json::Object(_)) => data.cloned().expect("mark data is present"), + Some(value) => { + let mut object = serde_json::Map::new(); + object.insert("value".to_string(), value.clone()); + Json::Object(object) + } + } +} + // A runtime mark is point-in-time telemetry rather than a scoped call with start/end events. Agent // hook adapters use marks for lifecycle notifications that do not map to first-class ATIF step // types, for example hook-only status updates or synthetic fallback events. The ATIF step message diff --git a/crates/core/tests/unit/atif_tests.rs b/crates/core/tests/unit/atif_tests.rs index 96acdfef0..e6a52230c 100644 --- a/crates/core/tests/unit/atif_tests.rs +++ b/crates/core/tests/unit/atif_tests.rs @@ -1963,6 +1963,15 @@ fn test_exporter_tool_projection_keeps_data_less_and_empty_marks_visible() { None, None, )); + let mut scalar_mark = Event::Mark(MarkEvent::new( + BaseEvent::builder() + .parent_uuid(root_uuid) + .name("plugin.scalar_checkpoint") + .data(json!("checkpoint complete")) + .build(), + None, + None, + )); let mut root_end = event_builder(root_uuid, EventType::End) .name("neutral-agent") .scope_type(ScopeType::Agent) @@ -1973,18 +1982,20 @@ fn test_exporter_tool_projection_keeps_data_less_and_empty_marks_visible() { base + chrono::Duration::milliseconds(1), ); set_event_timestamp(&mut empty_mark, base + chrono::Duration::milliseconds(2)); - set_event_timestamp(&mut root_end, base + chrono::Duration::milliseconds(3)); + set_event_timestamp(&mut scalar_mark, base + chrono::Duration::milliseconds(3)); + set_event_timestamp(&mut root_end, base + chrono::Duration::milliseconds(4)); exporter.state.lock().unwrap().events.extend([ root_start, data_less_mark, empty_mark, + scalar_mark, root_end, ]); let trajectory = exporter.export().unwrap(); assert_atif_v17_shape(&trajectory); - assert_eq!(trajectory.steps.len(), 2); + assert_eq!(trajectory.steps.len(), 3); let data_less_tool = &trajectory.steps[0].tool_calls.as_ref().unwrap()[0]; assert_eq!(data_less_tool.function_name, "plugin.checkpoint"); @@ -1999,6 +2010,19 @@ fn test_exporter_tool_projection_keeps_data_less_and_empty_marks_visible() { let empty_extra: AtifStepExtra = serde_json::from_value(trajectory.steps[1].extra.clone().unwrap()).unwrap(); assert_eq!(empty_extra.event_payload, Some(json!({}))); + + let scalar_tool = &trajectory.steps[2].tool_calls.as_ref().unwrap()[0]; + assert_eq!(scalar_tool.function_name, "plugin.scalar_checkpoint"); + assert_eq!( + scalar_tool.arguments, + json!({"value": "checkpoint complete"}) + ); + let scalar_extra: AtifStepExtra = + serde_json::from_value(trajectory.steps[2].extra.clone().unwrap()).unwrap(); + assert_eq!( + scalar_extra.event_payload, + Some(json!("checkpoint complete")) + ); } #[test] From d50754d04dfff092b9c23931a066aa7566b62ff7 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 7 Jul 2026 15:58:04 -0600 Subject: [PATCH 04/14] test(observability): cover projected mark metadata Signed-off-by: Bryan Bednarski --- .../tests/integration/middleware_tests.rs | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/crates/core/tests/integration/middleware_tests.rs b/crates/core/tests/integration/middleware_tests.rs index 866dbb447..0b184ec72 100644 --- a/crates/core/tests/integration/middleware_tests.rs +++ b/crates/core/tests/integration/middleware_tests.rs @@ -58,7 +58,7 @@ use nemo_relay::error::FlowError; #[cfg(all(feature = "otel", feature = "openinference"))] use nemo_relay::observability::MarkProjection; #[cfg(all(feature = "otel", feature = "openinference"))] -use nemo_relay::observability::atif::{AtifAgentInfo, AtifExporter}; +use nemo_relay::observability::atif::{AtifAgentInfo, AtifExporter, AtifStepExtra}; #[cfg(all(feature = "otel", feature = "openinference"))] use nemo_relay::observability::openinference::OpenInferenceSubscriber; #[cfg(all(feature = "otel", feature = "openinference"))] @@ -924,6 +924,17 @@ async fn test_managed_tool_pending_marks_project_through_all_exporters() { assert_eq!(mark_step.llm_call_count, Some(0)); let projected_call = &mark_step.tool_calls.as_ref().unwrap()[0]; assert_eq!(projected_call.arguments, json!({"saved_tokens": 12})); + let step_extra: AtifStepExtra = + serde_json::from_value(mark_step.extra.clone().unwrap()).unwrap(); + assert_eq!(step_extra.event_category.as_deref(), Some("custom")); + assert_eq!( + step_extra.event_category_profile, + Some(json!({"subtype": "example.compaction"})) + ); + assert_eq!( + projected_call.extra.as_ref().unwrap()["metadata"], + json!({"source": "test"}) + ); assert_eq!( mark_step.observation.as_ref().unwrap().results[0] .source_call_id @@ -947,6 +958,18 @@ async fn test_managed_tool_pending_marks_project_through_all_exporters() { attribute.key.as_str() == "nemo_relay.mark.projection" && attribute.value.to_string() == "tool" })); + for (key, value) in [ + ("nemo_relay.mark.category", "custom"), + ( + "nemo_relay.mark.category_profile_json", + "{\"subtype\":\"example.compaction\"}", + ), + ("nemo_relay.mark.metadata_json", "{\"source\":\"test\"}"), + ] { + assert!(otel_mark.attributes.iter().any(|attribute| { + attribute.key.as_str() == key && attribute.value.to_string() == value + })); + } openinference.force_flush().unwrap(); let openinference_spans = openinference_exporter.get_finished_spans().unwrap(); @@ -966,6 +989,18 @@ async fn test_managed_tool_pending_marks_project_through_all_exporters() { assert!(openinference_mark.attributes.iter().any(|attribute| { attribute.key.as_str() == "openinference.span.kind" && attribute.value.to_string() == "TOOL" })); + for (key, value) in [ + ("nemo_relay.mark.category", "custom"), + ( + "nemo_relay.mark.category_profile_json", + "{\"subtype\":\"example.compaction\"}", + ), + ("nemo_relay.mark.metadata_json", "{\"source\":\"test\"}"), + ] { + assert!(openinference_mark.attributes.iter().any(|attribute| { + attribute.key.as_str() == key && attribute.value.to_string() == value + })); + } deregister_tool_execution_intercept("managed_tool_projection_intercept").unwrap(); deregister_subscriber("managed_tool_projection_events").unwrap(); From c9b8d8665add37701cba4e59123d33a902cb3ec8 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 7 Jul 2026 16:44:05 -0600 Subject: [PATCH 05/14] feat(observability): configure mark projection exclusions Signed-off-by: Bryan Bednarski --- crates/core/src/observability/atif.rs | 51 +++++++++++-- crates/core/src/observability/mod.rs | 29 ++++++- .../core/src/observability/openinference.rs | 65 +++++++++++++++- crates/core/src/observability/otel.rs | 75 +++++++++++++++++-- .../src/observability/plugin_component.rs | 25 ++++++- crates/core/tests/unit/atif_tests.rs | 32 ++++++++ .../unit/observability/openinference_tests.rs | 29 +++++++ .../tests/unit/observability/otel_tests.rs | 29 +++++++ .../observability/plugin_component_tests.rs | 13 ++++ crates/node/observability.d.ts | 2 + crates/node/observability.js | 2 + .../node/tests/observability_plugin_tests.mjs | 2 + docs/configure-plugins/observability/atif.mdx | 7 ++ .../observability/openinference.mdx | 5 ++ .../observability/opentelemetry.mdx | 5 ++ go/nemo_relay/observability_plugin.go | 4 + go/nemo_relay/observability_plugin_test.go | 4 +- python/nemo_relay/observability.py | 4 + python/nemo_relay/observability.pyi | 2 + python/tests/test_observability_plugin.py | 2 + 20 files changed, 360 insertions(+), 27 deletions(-) diff --git a/crates/core/src/observability/atif.rs b/crates/core/src/observability/atif.rs index 423ce3468..216eb19bc 100644 --- a/crates/core/src/observability/atif.rs +++ b/crates/core/src/observability/atif.rs @@ -44,8 +44,8 @@ use crate::error::Result; use crate::json::Json; use super::{ - MarkProjection, estimate_cost_for_response_or_model, is_llm_chunk_mark, manual, merge_usage, - model_name_for_llm_event, + MarkProjection, default_mark_exclude_names, estimate_cost_for_response_or_model, + is_llm_chunk_mark, manual, mark_name_is_excluded, merge_usage, model_name_for_llm_event, }; /// The ATIF schema version string embedded in all exported trajectories. @@ -332,6 +332,7 @@ struct AtifExporterState { session_id: String, agent_info: AtifAgentInfo, mark_projection: MarkProjection, + mark_exclude_names: Vec, events: Vec, } @@ -359,6 +360,7 @@ impl AtifExporter { session_id, agent_info, mark_projection: MarkProjection::default(), + mark_exclude_names: default_mark_exclude_names(), events: Vec::new(), })), } @@ -374,6 +376,18 @@ impl AtifExporter { self } + /// Excludes named marks from tool projection while preserving their native + /// event representation. The default excludes high-volume `llm.chunk` + /// marks. + pub fn with_mark_exclude_names(self, names: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.state.lock().unwrap().mark_exclude_names = names.into_iter().map(Into::into).collect(); + self + } + /// Return an event subscriber function that records NeMo Relay events. /// /// The returned callback can be registered with @@ -413,12 +427,13 @@ impl AtifExporter { /// callers that prefer an explicitly fallible method name. pub fn try_export(&self) -> Result { flush_subscribers()?; - let (session_id, agent_info, mark_projection, events) = { + let (session_id, agent_info, mark_projection, mark_exclude_names, events) = { let state = self.state.lock().unwrap(); ( state.session_id.clone(), state.agent_info.clone(), state.mark_projection, + state.mark_exclude_names.clone(), state.events.clone(), ) }; @@ -427,6 +442,7 @@ impl AtifExporter { &session_id, agent_info, mark_projection, + &mark_exclude_names, &collected_events, )) } @@ -1991,6 +2007,7 @@ impl PendingAgentStep { #[derive(Default)] struct StepConversionState { mark_projection: MarkProjection, + mark_exclude_names: Vec, steps: Vec, last_tool_call_map: std::collections::HashMap, tool_scope_call_ids: std::collections::HashMap, @@ -2620,7 +2637,9 @@ impl StepConversionState { return; } self.flush_observations(); - if self.mark_projection == MarkProjection::Tool { + if self.mark_projection == MarkProjection::Tool + && !mark_name_is_excluded(mark, &self.mark_exclude_names) + { self.handle_mark_as_tool(mark, lookups, mark.data()); return; } @@ -3280,6 +3299,7 @@ fn events_to_trajectory( session_id: &str, agent_info: AtifAgentInfo, mark_projection: MarkProjection, + mark_exclude_names: &[String], events: &[&Event], ) -> AtifTrajectory { let mut sorted: Vec<&Event> = events.to_vec(); @@ -3295,12 +3315,13 @@ fn events_to_trajectory( session_id, &agent_info, mark_projection, + mark_exclude_names, &sorted, true, ); } - let steps = events_to_steps(&sorted, mark_projection); + let steps = events_to_steps(&sorted, mark_projection, mark_exclude_names); trajectory_from_parts( session_id.to_string(), Some(session_id.to_string()), @@ -3339,16 +3360,24 @@ fn is_step_event(event: &Event) -> bool { ) } +#[allow(clippy::too_many_arguments)] fn agent_scope_to_trajectory( tree: &AgentScopeTree, agent_uuid: Uuid, session_id: &str, agent_info: &AtifAgentInfo, mark_projection: MarkProjection, + mark_exclude_names: &[String], sorted_events: &[&Event], is_root: bool, ) -> AtifTrajectory { - let mut steps = events_to_steps_for_agent(sorted_events, tree, agent_uuid, mark_projection); + let mut steps = events_to_steps_for_agent( + sorted_events, + tree, + agent_uuid, + mark_projection, + mark_exclude_names, + ); let subagent_trajectories = tree .nodes .get(&agent_uuid) @@ -3362,6 +3391,7 @@ fn agent_scope_to_trajectory( session_id, agent_info, mark_projection, + mark_exclude_names, sorted_events, false, ) @@ -3436,10 +3466,12 @@ fn events_to_steps_for_agent( tree: &AgentScopeTree, agent_uuid: Uuid, mark_projection: MarkProjection, + mark_exclude_names: &[String], ) -> Vec { let lookups = EventLookupMaps::from_events_for_agent(events, tree, agent_uuid); let mut state = StepConversionState { mark_projection, + mark_exclude_names: mark_exclude_names.to_vec(), ..StepConversionState::default() }; @@ -3477,12 +3509,17 @@ fn events_to_steps_for_agent( /// promoted tool_calls by function name → `source_call_id`. /// 5. Mark events → system steps if they carry data. /// 6. Scope Start/End → skipped. -fn events_to_steps(events: &[&Event], mark_projection: MarkProjection) -> Vec { +fn events_to_steps( + events: &[&Event], + mark_projection: MarkProjection, + mark_exclude_names: &[String], +) -> Vec { let mut sorted: Vec<&Event> = events.to_vec(); sorted.sort_by_key(|e| *e.timestamp()); let lookups = EventLookupMaps::from_events(&sorted); let mut state = StepConversionState { mark_projection, + mark_exclude_names: mark_exclude_names.to_vec(), ..StepConversionState::default() }; diff --git a/crates/core/src/observability/mod.rs b/crates/core/src/observability/mod.rs index 122f8615a..724f8c142 100644 --- a/crates/core/src/observability/mod.rs +++ b/crates/core/src/observability/mod.rs @@ -35,12 +35,37 @@ pub enum MarkProjection { /// Preserve marks as ATIF system steps or OTEL span events. #[default] Event, - /// Render non-streaming marks as deterministic ATIF tool steps or + /// Render non-excluded marks as deterministic ATIF tool steps or /// zero-duration OTEL child spans so trace-tree consumers can display them - /// directly. High-volume `llm.chunk` marks remain events. + /// directly. High-volume `llm.chunk` marks remain exporter-native events. Tool, } +/// Default mark names excluded from tool projection because they are emitted +/// at high volume and are better represented as exporter-native events. +pub(crate) fn default_mark_exclude_names() -> Vec { + vec!["llm.chunk".to_string()] +} + +/// Returns whether a mark matches a configured projection exclusion. +/// +/// Agent hook adapters may preserve the canonical event name in metadata while +/// using a generic mark name, so both representations are matched. +pub(crate) fn mark_name_is_excluded( + event: &crate::api::event::Event, + excluded_names: &[String], +) -> bool { + excluded_names.iter().any(|name| { + event.name() == name + || event + .metadata() + .and_then(crate::json::Json::as_object) + .and_then(|metadata| metadata.get("hook_event_name")) + .and_then(crate::json::Json::as_str) + == Some(name.as_str()) + }) +} + pub(crate) fn is_llm_chunk_mark(event: &crate::api::event::Event) -> bool { event.name() == "llm.chunk" || event diff --git a/crates/core/src/observability/openinference.rs b/crates/core/src/observability/openinference.rs index 90de8b65a..1f9867207 100644 --- a/crates/core/src/observability/openinference.rs +++ b/crates/core/src/observability/openinference.rs @@ -21,8 +21,8 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use super::{ - MarkProjection, estimate_cost_for_response_or_model, - estimate_cost_for_response_or_requested_model, is_llm_chunk_mark, manual, merge_usage, + MarkProjection, default_mark_exclude_names, estimate_cost_for_response_or_model, + estimate_cost_for_response_or_requested_model, manual, mark_name_is_excluded, merge_usage, model_name_for_llm_event, }; use crate::api::event::{Event, EventNormalizationExt, ScopeCategory}; @@ -103,6 +103,7 @@ pub struct OpenInferenceConfig { service_version: Option, instrumentation_scope: String, mark_projection: MarkProjection, + mark_exclude_names: Vec, timeout: Duration, transport: OtlpTransport, } @@ -118,6 +119,7 @@ impl Default for OpenInferenceConfig { service_version: None, instrumentation_scope: "nemo-relay-openinference".to_string(), mark_projection: MarkProjection::default(), + mark_exclude_names: default_mark_exclude_names(), timeout: Duration::from_secs(3), transport: OtlpTransport::HttpBinary, } @@ -193,6 +195,18 @@ impl OpenInferenceConfig { self.mark_projection = mark_projection; self } + + /// Excludes named marks from tool projection while preserving their native + /// event representation. The default excludes high-volume `llm.chunk` + /// marks. + pub fn with_mark_exclude_names(mut self, names: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.mark_exclude_names = names.into_iter().map(Into::into).collect(); + self + } } /// OpenInference-backed NeMo Relay subscriber. @@ -219,6 +233,7 @@ impl OpenInferenceSubscriber { provider, config.instrumentation_scope, config.mark_projection, + config.mark_exclude_names, )) } @@ -231,6 +246,7 @@ impl OpenInferenceSubscriber { provider, instrumentation_scope.into(), MarkProjection::default(), + default_mark_exclude_names(), ) } @@ -244,6 +260,26 @@ impl OpenInferenceSubscriber { provider, instrumentation_scope.into(), mark_projection, + default_mark_exclude_names(), + ) + } + + /// Builds a subscriber with explicit mark projection and exclusion names. + pub fn from_tracer_provider_with_mark_projection_and_exclusions( + provider: SdkTracerProvider, + instrumentation_scope: impl Into, + mark_projection: MarkProjection, + mark_exclude_names: I, + ) -> Self + where + I: IntoIterator, + S: Into, + { + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + mark_projection, + mark_exclude_names.into_iter().map(Into::into).collect(), ) } @@ -251,12 +287,14 @@ impl OpenInferenceSubscriber { provider: SdkTracerProvider, instrumentation_scope: String, mark_projection: MarkProjection, + mark_exclude_names: Vec, ) -> Self { let processor = Arc::new(Mutex::new( - OpenInferenceEventProcessor::new_with_mark_projection( + OpenInferenceEventProcessor::new_with_mark_projection_and_exclusions( provider, instrumentation_scope, mark_projection, + mark_exclude_names, ), )); let processor_for_callback = Arc::clone(&processor); @@ -413,6 +451,7 @@ struct OpenInferenceEventProcessor { provider: SdkTracerProvider, tracer: SdkTracer, mark_projection: MarkProjection, + mark_exclude_names: Vec, } impl OpenInferenceEventProcessor { @@ -421,10 +460,25 @@ impl OpenInferenceEventProcessor { Self::new_with_mark_projection(provider, instrumentation_scope, MarkProjection::default()) } + #[cfg(test)] fn new_with_mark_projection( provider: SdkTracerProvider, instrumentation_scope: String, mark_projection: MarkProjection, + ) -> Self { + Self::new_with_mark_projection_and_exclusions( + provider, + instrumentation_scope, + mark_projection, + default_mark_exclude_names(), + ) + } + + fn new_with_mark_projection_and_exclusions( + provider: SdkTracerProvider, + instrumentation_scope: String, + mark_projection: MarkProjection, + mark_exclude_names: Vec, ) -> Self { let tracer = provider.tracer(instrumentation_scope); Self { @@ -434,6 +488,7 @@ impl OpenInferenceEventProcessor { provider, tracer, mark_projection, + mark_exclude_names, } } @@ -484,7 +539,9 @@ impl OpenInferenceEventProcessor { } fn process_mark(&mut self, event: &Event) { - if self.mark_projection == MarkProjection::Tool && !is_llm_chunk_mark(event) { + if self.mark_projection == MarkProjection::Tool + && !mark_name_is_excluded(event, &self.mark_exclude_names) + { self.process_mark_as_tool(event); return; } diff --git a/crates/core/src/observability/otel.rs b/crates/core/src/observability/otel.rs index 1bc6b2b24..ce277db3c 100644 --- a/crates/core/src/observability/otel.rs +++ b/crates/core/src/observability/otel.rs @@ -21,8 +21,8 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use super::{ - MarkProjection, estimate_cost_for_response_or_model, - estimate_cost_for_response_or_requested_model, is_llm_chunk_mark, manual, + MarkProjection, default_mark_exclude_names, estimate_cost_for_response_or_model, + estimate_cost_for_response_or_requested_model, manual, mark_name_is_excluded, model_name_for_llm_event, }; use crate::api::event::{Event, EventNormalizationExt, ScopeCategory}; @@ -97,6 +97,7 @@ pub struct OpenTelemetryConfig { service_version: Option, instrumentation_scope: String, mark_projection: MarkProjection, + mark_exclude_names: Vec, timeout: Duration, transport: OtlpTransport, } @@ -112,6 +113,7 @@ impl Default for OpenTelemetryConfig { service_version: None, instrumentation_scope: "nemo-relay-otel".to_string(), mark_projection: MarkProjection::default(), + mark_exclude_names: default_mark_exclude_names(), timeout: Duration::from_secs(3), transport: OtlpTransport::HttpBinary, } @@ -188,6 +190,18 @@ impl OpenTelemetryConfig { self.mark_projection = mark_projection; self } + + /// Excludes named marks from tool projection while preserving their native + /// event representation. The default excludes high-volume `llm.chunk` + /// marks. + pub fn with_mark_exclude_names(mut self, names: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.mark_exclude_names = names.into_iter().map(Into::into).collect(); + self + } } /// OpenTelemetry-backed NeMo Relay subscriber. @@ -214,6 +228,7 @@ impl OpenTelemetrySubscriber { provider, config.instrumentation_scope, config.mark_projection, + config.mark_exclude_names, )) } @@ -226,6 +241,7 @@ impl OpenTelemetrySubscriber { provider, instrumentation_scope.into(), MarkProjection::default(), + default_mark_exclude_names(), ) } @@ -239,6 +255,26 @@ impl OpenTelemetrySubscriber { provider, instrumentation_scope.into(), mark_projection, + default_mark_exclude_names(), + ) + } + + /// Builds a subscriber with explicit mark projection and exclusion names. + pub fn from_tracer_provider_with_mark_projection_and_exclusions( + provider: SdkTracerProvider, + instrumentation_scope: impl Into, + mark_projection: MarkProjection, + mark_exclude_names: I, + ) -> Self + where + I: IntoIterator, + S: Into, + { + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + mark_projection, + mark_exclude_names.into_iter().map(Into::into).collect(), ) } @@ -246,12 +282,16 @@ impl OpenTelemetrySubscriber { provider: SdkTracerProvider, instrumentation_scope: String, mark_projection: MarkProjection, + mark_exclude_names: Vec, ) -> Self { - let processor = Arc::new(Mutex::new(OtelEventProcessor::new_with_mark_projection( - provider, - instrumentation_scope, - mark_projection, - ))); + let processor = Arc::new(Mutex::new( + OtelEventProcessor::new_with_mark_projection_and_exclusions( + provider, + instrumentation_scope, + mark_projection, + mark_exclude_names, + ), + )); let processor_for_callback = Arc::clone(&processor); let subscriber: EventSubscriberFn = Arc::new(move |event: &Event| { let Ok(mut guard) = processor_for_callback.lock() else { @@ -405,6 +445,7 @@ struct OtelEventProcessor { provider: SdkTracerProvider, tracer: SdkTracer, mark_projection: MarkProjection, + mark_exclude_names: Vec, } impl OtelEventProcessor { @@ -413,10 +454,25 @@ impl OtelEventProcessor { Self::new_with_mark_projection(provider, instrumentation_scope, MarkProjection::default()) } + #[cfg(test)] fn new_with_mark_projection( provider: SdkTracerProvider, instrumentation_scope: String, mark_projection: MarkProjection, + ) -> Self { + Self::new_with_mark_projection_and_exclusions( + provider, + instrumentation_scope, + mark_projection, + default_mark_exclude_names(), + ) + } + + fn new_with_mark_projection_and_exclusions( + provider: SdkTracerProvider, + instrumentation_scope: String, + mark_projection: MarkProjection, + mark_exclude_names: Vec, ) -> Self { let tracer = provider.tracer(instrumentation_scope); Self { @@ -426,6 +482,7 @@ impl OtelEventProcessor { provider, tracer, mark_projection, + mark_exclude_names, } } @@ -477,7 +534,9 @@ impl OtelEventProcessor { } fn process_mark(&mut self, event: &Event) { - if self.mark_projection == MarkProjection::Tool && !is_llm_chunk_mark(event) { + if self.mark_projection == MarkProjection::Tool + && !mark_name_is_excluded(event, &self.mark_exclude_names) + { self.process_mark_as_tool(event); return; } diff --git a/crates/core/src/observability/plugin_component.rs b/crates/core/src/observability/plugin_component.rs index 87e4c6cdd..d17b0ab61 100644 --- a/crates/core/src/observability/plugin_component.rs +++ b/crates/core/src/observability/plugin_component.rs @@ -36,7 +36,6 @@ use crate::api::subscriber::{ scope_deregister_subscriber, try_scope_deregister_subscriber, try_scope_register_subscriber, }; use crate::error::FlowError; -use crate::observability::MarkProjection; use crate::observability::atif::{AtifAgentInfo, AtifExporter}; use crate::observability::atof::{ AtofEndpointConfig as CoreAtofEndpointConfig, AtofEndpointFieldNamePolicy, @@ -52,6 +51,7 @@ use crate::observability::openinference::{ use crate::observability::otel::{ OpenTelemetryConfig as CoreOpenTelemetryConfig, OpenTelemetrySubscriber, }; +use crate::observability::{MarkProjection, default_mark_exclude_names}; use crate::plugin::{ ConfigDiagnostic, ConfigPolicy, DiagnosticLevel, Plugin, PluginComponentSpec, PluginError, PluginRegistration, PluginRegistrationContext, Result as PluginResult, UnsupportedBehavior, @@ -232,6 +232,9 @@ pub struct AtifSectionConfig { #[serde(default)] #[cfg_attr(feature = "schema", schemars(schema_with = "mark_projection_schema"))] pub mark_projection: MarkProjection, + /// Mark names excluded from tool projection. Defaults to `llm.chunk`. + #[serde(default = "default_mark_exclude_names")] + pub mark_exclude_names: Vec, /// Tool definitions available to the agent. #[serde(default, skip_serializing_if = "Option::is_none")] pub tool_definitions: Option>, @@ -269,6 +272,7 @@ impl Default for AtifSectionConfig { agent_version: default_agent_version(), model_name: default_model_name(), mark_projection: MarkProjection::default(), + mark_exclude_names: default_mark_exclude_names(), tool_definitions: None, extra: None, output_directory: None, @@ -384,6 +388,9 @@ pub struct OtlpSectionConfig { #[serde(default)] #[cfg_attr(feature = "schema", schemars(schema_with = "mark_projection_schema"))] pub mark_projection: MarkProjection, + /// Mark names excluded from tool projection. Defaults to `llm.chunk`. + #[serde(default = "default_mark_exclude_names")] + pub mark_exclude_names: Vec, /// OTLP transport: `http_binary` or `grpc`. #[serde(default = "default_otlp_transport")] #[cfg_attr(feature = "schema", schemars(schema_with = "otlp_transport_schema"))] @@ -419,6 +426,7 @@ impl Default for OtlpSectionConfig { Self { enabled: false, mark_projection: MarkProjection::default(), + mark_exclude_names: default_mark_exclude_names(), transport: default_otlp_transport(), endpoint: None, headers: HashMap::new(), @@ -488,6 +496,7 @@ crate::editor_config! { agent_version => { label: "agent_version", kind: String }, model_name => { label: "model_name", kind: String }, mark_projection => { label: "mark_projection", kind: Enum, values: ["event", "tool"] }, + mark_exclude_names => { label: "mark_exclude_names", kind: Json }, tool_definitions => { label: "tool_definitions", kind: Json, optional: true }, extra => { label: "extra", kind: Json, optional: true }, output_directory => { label: "output_directory", kind: String, optional: true }, @@ -500,6 +509,7 @@ crate::editor_config! { impl OtlpSectionConfig { enabled => { label: "enabled", kind: Boolean }, mark_projection => { label: "mark_projection", kind: Enum, values: ["event", "tool"] }, + mark_exclude_names => { label: "mark_exclude_names", kind: Json }, transport => { label: "transport", kind: Enum, values: ["http_binary", "grpc"] }, endpoint => { label: "endpoint", kind: String, optional: true }, headers => { label: "headers", kind: StringMap }, @@ -950,7 +960,8 @@ impl AtifDispatcher { // emitted. let session_id = event.uuid().to_string(); let exporter = AtifExporter::new(session_id.clone(), self.agent_info()) - .with_mark_projection(self.config.mark_projection); + .with_mark_projection(self.config.mark_projection) + .with_mark_exclude_names(self.config.mark_exclude_names.clone()); (exporter.subscriber())(event); let (filename, local_path) = self.prepare_destination(&session_id); self.scope_owners.insert(event.uuid(), event.uuid()); @@ -1368,7 +1379,9 @@ fn build_otel_config(section: OtlpSectionConfig) -> PluginResult PluginResult(json!({ "mark_projection": "span" })) @@ -288,6 +300,7 @@ fn schema_contains_every_supported_observability_option() { "agent_version", "model_name", "mark_projection", + "mark_exclude_names", "tool_definitions", "extra", "filename_template", diff --git a/crates/node/observability.d.ts b/crates/node/observability.d.ts index 1d0201a00..70ea82f82 100644 --- a/crates/node/observability.d.ts +++ b/crates/node/observability.d.ts @@ -48,6 +48,7 @@ export interface AtifConfig { agent_version?: string; model_name?: string; mark_projection?: 'event' | 'tool'; + mark_exclude_names?: string[]; tool_definitions?: Record[]; extra?: Record; output_directory?: string; @@ -58,6 +59,7 @@ export interface AtifConfig { export interface OtlpConfig { enabled?: boolean; mark_projection?: 'event' | 'tool'; + mark_exclude_names?: string[]; transport?: 'http_binary' | 'grpc' | string; endpoint?: string; headers?: Record; diff --git a/crates/node/observability.js b/crates/node/observability.js index 637fa492b..47a875531 100644 --- a/crates/node/observability.js +++ b/crates/node/observability.js @@ -44,6 +44,7 @@ function atifConfig(config = {}) { agent_name: 'NeMo Relay', model_name: 'unknown', mark_projection: 'event', + mark_exclude_names: ['llm.chunk'], filename_template: 'nemo-relay-atif-{session_id}.json', ...config, }; @@ -59,6 +60,7 @@ function otlpConfig(config = {}) { return { enabled: false, mark_projection: 'event', + mark_exclude_names: ['llm.chunk'], transport: 'http_binary', headers: {}, resource_attributes: {}, diff --git a/crates/node/tests/observability_plugin_tests.mjs b/crates/node/tests/observability_plugin_tests.mjs index a5d91f2eb..d09b9c3cf 100644 --- a/crates/node/tests/observability_plugin_tests.mjs +++ b/crates/node/tests/observability_plugin_tests.mjs @@ -26,11 +26,13 @@ describe('observability plugin helpers', () => { agent_name: 'NeMo Relay', model_name: 'unknown', mark_projection: 'event', + mark_exclude_names: ['llm.chunk'], filename_template: 'nemo-relay-atif-{session_id}.json', }); assert.deepEqual(observability.otlpConfig(), { enabled: false, mark_projection: 'event', + mark_exclude_names: ['llm.chunk'], transport: 'http_binary', headers: {}, resource_attributes: {}, diff --git a/docs/configure-plugins/observability/atif.mdx b/docs/configure-plugins/observability/atif.mdx index 7d956e1ae..9af2ad413 100644 --- a/docs/configure-plugins/observability/atif.mdx +++ b/docs/configure-plugins/observability/atif.mdx @@ -54,6 +54,7 @@ The following table describes the top-level ATIF settings: | `agent_version` | NeMo Relay crate version | Agent version metadata. | | `model_name` | `unknown` | Default model metadata when no call-level model is present. | | `mark_projection` | `event` | `event` preserves marks as system steps. `tool` emits deterministic agent/tool steps for trajectory viewers that render tool calls but not system events. | +| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain native event handling. Metadata `hook_event_name` aliases are also matched. | | `tool_definitions` | Omitted | Optional ATIF tool metadata. | | `extra` | Omitted | Optional ATIF agent metadata. | | `output_directory` | Current working directory | Directory containing trajectory files. Ignored when `storage` is non-empty. | @@ -201,6 +202,12 @@ category, category profile, timestamp, and ancestry preserved in `extra`. Set projection. The projected step uses `llm_call_count = 0`, retains the original mark data in `extra`, and does not change the canonical ATOF event. +Add other event names to `mark_exclude_names` when a trajectory should keep +those marks in their native system-step representation rather than projecting +them as tool steps. The exclusion list affects only tool projection; it does +not remove mark payload or metadata. ATIF continues to suppress the built-in +high-volume `llm.chunk` stream receipts as it does in event mode. + The exporter translates NeMo Relay lifecycle events into ATIF v1.7 trajectory data. LLM start and end events become model steps, tool events become tool calls and observations, and scope nesting contributes lineage metadata. diff --git a/docs/configure-plugins/observability/openinference.mdx b/docs/configure-plugins/observability/openinference.mdx index e0542be8c..1efa3ad9c 100644 --- a/docs/configure-plugins/observability/openinference.mdx +++ b/docs/configure-plugins/observability/openinference.mdx @@ -62,6 +62,7 @@ OpenInference uses the same OTLP section shape as |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | | `mark_projection` | `event` | `event` attaches marks to active parent spans. `tool` emits zero-duration OpenInference `TOOL` child spans. | +| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain native event handling. Metadata `hook_event_name` aliases are also matched. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | | `headers` | `{}` | String-to-string exporter headers. | @@ -90,6 +91,10 @@ The default `event` projection preserves marks as span events. With profile, timestamp, and parent identifiers retained as attributes. High-volume `llm.chunk` marks remain span events. +Add other event names to `mark_exclude_names` when a backend should keep those +marks as native span events rather than visible tool children. The exclusion +list affects only tool projection; it does not remove mark payload or metadata. + Each lifecycle span includes `nemo_relay.uuid` and `nemo_relay.parent_uuid` attributes. These values match ATIF `step.extra.ancestry.function_id` and `step.extra.ancestry.parent_id` for the same events. For plugin-managed ATIF, diff --git a/docs/configure-plugins/observability/opentelemetry.mdx b/docs/configure-plugins/observability/opentelemetry.mdx index 38eb6c97a..95e23088f 100644 --- a/docs/configure-plugins/observability/opentelemetry.mdx +++ b/docs/configure-plugins/observability/opentelemetry.mdx @@ -56,6 +56,7 @@ The following table describes OpenTelemetry exporter settings: |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | | `mark_projection` | `event` | `event` attaches marks to active parent spans. `tool` emits zero-duration child spans for trace-tree visibility. | +| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain native event handling. Metadata `hook_event_name` aliases are also matched. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | | `headers` | `{}` | String-to-string exporter headers. | @@ -79,6 +80,10 @@ marks represented as visible child spans. Projected marks use `SpanKind::Interna category/profile, and payload attributes. High-volume `llm.chunk` marks remain span events. +Add other event names to `mark_exclude_names` when a backend should keep those +marks as native span events rather than visible tool children. The exclusion +list affects only tool projection; it does not remove mark payload or metadata. + Each lifecycle span includes `nemo_relay.uuid` and `nemo_relay.parent_uuid` attributes. These values match ATIF `step.extra.ancestry.function_id` and `step.extra.ancestry.parent_id` for the same events. For plugin-managed ATIF, diff --git a/go/nemo_relay/observability_plugin.go b/go/nemo_relay/observability_plugin.go index 0bb411fed..2e481e62c 100644 --- a/go/nemo_relay/observability_plugin.go +++ b/go/nemo_relay/observability_plugin.go @@ -53,6 +53,7 @@ type ObservabilityAtifConfig struct { AgentVersion string `json:"agent_version,omitempty"` ModelName string `json:"model_name,omitempty"` MarkProjection ObservabilityMarkProjection `json:"mark_projection,omitempty"` + MarkExcludeNames []string `json:"mark_exclude_names,omitempty"` ToolDefinitions []map[string]any `json:"tool_definitions,omitempty"` Extra map[string]any `json:"extra,omitempty"` OutputDirectory string `json:"output_directory,omitempty"` @@ -141,6 +142,7 @@ func (config ObservabilityHttpStorageConfig) MarshalJSON() ([]byte, error) { type ObservabilityOtlpConfig struct { Enabled bool `json:"enabled,omitempty"` MarkProjection ObservabilityMarkProjection `json:"mark_projection,omitempty"` + MarkExcludeNames []string `json:"mark_exclude_names,omitempty"` Transport string `json:"transport,omitempty"` Endpoint string `json:"endpoint,omitempty"` Headers map[string]string `json:"headers,omitempty"` @@ -176,6 +178,7 @@ func NewObservabilityAtifConfig() ObservabilityAtifConfig { AgentName: "NeMo Relay", ModelName: "unknown", MarkProjection: ObservabilityMarkProjectionEvent, + MarkExcludeNames: []string{"llm.chunk"}, FilenameTemplate: "nemo-relay-atif-{session_id}.json", } } @@ -195,6 +198,7 @@ func NewObservabilityOtlpConfig() ObservabilityOtlpConfig { return ObservabilityOtlpConfig{ Transport: "http_binary", MarkProjection: ObservabilityMarkProjectionEvent, + MarkExcludeNames: []string{"llm.chunk"}, Headers: map[string]string{}, ResourceAttributes: map[string]string{}, ServiceName: "nemo-relay", diff --git a/go/nemo_relay/observability_plugin_test.go b/go/nemo_relay/observability_plugin_test.go index b3d28901e..62533dc6f 100644 --- a/go/nemo_relay/observability_plugin_test.go +++ b/go/nemo_relay/observability_plugin_test.go @@ -43,7 +43,7 @@ func TestObservabilityConfigHelpers(t *testing.T) { FieldNamePolicy: "replace_dots", }} atif := NewObservabilityAtifConfig() - if atif.Enabled || atif.AgentName != "NeMo Relay" || atif.ModelName != "unknown" || atif.MarkProjection != ObservabilityMarkProjectionEvent || atif.FilenameTemplate != "nemo-relay-atif-{session_id}.json" { + if atif.Enabled || atif.AgentName != "NeMo Relay" || atif.ModelName != "unknown" || atif.MarkProjection != ObservabilityMarkProjectionEvent || len(atif.MarkExcludeNames) != 1 || atif.MarkExcludeNames[0] != "llm.chunk" || atif.FilenameTemplate != "nemo-relay-atif-{session_id}.json" { t.Fatalf("unexpected ATIF defaults: %#v", atif) } allowHTTP := false @@ -64,7 +64,7 @@ func TestObservabilityConfigHelpers(t *testing.T) { httpStorage, } otlp := NewObservabilityOtlpConfig() - if otlp.Enabled || otlp.MarkProjection != ObservabilityMarkProjectionEvent || otlp.Transport != "http_binary" || otlp.ServiceName != "nemo-relay" || otlp.TimeoutMillis != 3000 { + if otlp.Enabled || otlp.MarkProjection != ObservabilityMarkProjectionEvent || len(otlp.MarkExcludeNames) != 1 || otlp.MarkExcludeNames[0] != "llm.chunk" || otlp.Transport != "http_binary" || otlp.ServiceName != "nemo-relay" || otlp.TimeoutMillis != 3000 { t.Fatalf("unexpected OTLP defaults: %#v", otlp) } atif.MarkProjection = ObservabilityMarkProjectionTool diff --git a/python/nemo_relay/observability.py b/python/nemo_relay/observability.py index f57edb34b..f565a491f 100644 --- a/python/nemo_relay/observability.py +++ b/python/nemo_relay/observability.py @@ -168,6 +168,7 @@ class AtifConfig: agent_version: str | None = None model_name: str = "unknown" mark_projection: MarkProjection = "event" + mark_exclude_names: list[str] = field(default_factory=lambda: ["llm.chunk"]) tool_definitions: list[JsonObject] | None = None extra: JsonObject | None = None output_directory: str | None = None @@ -182,6 +183,7 @@ def to_dict(self) -> JsonObject: "agent_version": self.agent_version, "model_name": self.model_name, "mark_projection": self.mark_projection, + "mark_exclude_names": self.mark_exclude_names, "tool_definitions": self.tool_definitions, "extra": self.extra, "output_directory": self.output_directory, @@ -199,6 +201,7 @@ class OtlpConfig: enabled: bool = False mark_projection: MarkProjection = "event" + mark_exclude_names: list[str] = field(default_factory=lambda: ["llm.chunk"]) transport: Literal["http_binary", "grpc"] = "http_binary" endpoint: str | None = None headers: dict[str, str] = field(default_factory=dict) @@ -215,6 +218,7 @@ def to_dict(self) -> JsonObject: { "enabled": self.enabled, "mark_projection": self.mark_projection, + "mark_exclude_names": self.mark_exclude_names, "transport": self.transport, "endpoint": self.endpoint, "headers": self.headers, diff --git a/python/nemo_relay/observability.pyi b/python/nemo_relay/observability.pyi index a117e0d52..61c5af8af 100644 --- a/python/nemo_relay/observability.pyi +++ b/python/nemo_relay/observability.pyi @@ -64,6 +64,7 @@ class AtifConfig: agent_version: str | None = ... model_name: str = ... mark_projection: MarkProjection = ... + mark_exclude_names: list[str] = ... tool_definitions: list[JsonObject] | None = ... extra: JsonObject | None = ... output_directory: str | None = ... @@ -75,6 +76,7 @@ class AtifConfig: class OtlpConfig: enabled: bool = ... mark_projection: MarkProjection = ... + mark_exclude_names: list[str] = ... transport: Literal["http_binary", "grpc"] = ... endpoint: str | None = ... headers: dict[str, str] = field(default_factory=dict) diff --git a/python/tests/test_observability_plugin.py b/python/tests/test_observability_plugin.py index 2befca774..60378ce76 100644 --- a/python/tests/test_observability_plugin.py +++ b/python/tests/test_observability_plugin.py @@ -35,11 +35,13 @@ def test_defaults_and_component_wrapper(self): "agent_name": "NeMo Relay", "model_name": "unknown", "mark_projection": "event", + "mark_exclude_names": ["llm.chunk"], "filename_template": "nemo-relay-atif-{session_id}.json", } assert OtlpConfig().to_dict() == { "enabled": False, "mark_projection": "event", + "mark_exclude_names": ["llm.chunk"], "transport": "http_binary", "headers": {}, "resource_attributes": {}, From 6393bdd6767bca956a4c49227a16aad358e862f5 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 7 Jul 2026 16:45:34 -0600 Subject: [PATCH 06/14] test(observability): cover mark exclusion aliases Signed-off-by: Bryan Bednarski --- .../unit/observability/openinference_tests.rs | 14 +++++++++++++- crates/core/tests/unit/observability/otel_tests.rs | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/crates/core/tests/unit/observability/openinference_tests.rs b/crates/core/tests/unit/observability/openinference_tests.rs index 0564547a3..ca2b61311 100644 --- a/crates/core/tests/unit/observability/openinference_tests.rs +++ b/crates/core/tests/unit/observability/openinference_tests.rs @@ -2265,17 +2265,29 @@ fn tool_projection_keeps_llm_chunks_as_parent_span_events() { let parent_uuid = Uuid::now_v7(); let start = make_start_event(parent_uuid, None, "llm", ScopeType::Llm, None); let chunk = make_mark_event(Some(parent_uuid), "llm.chunk", Some(json!({"delta": "x"}))); + let aliased_chunk = Event::Mark(MarkEvent::new( + BaseEvent::builder() + .parent_uuid(parent_uuid) + .name("hook_mark") + .metadata(json!({"hook_event_name": "llm.chunk"})) + .data(json!({"delta": "y"})) + .build(), + None, + None, + )); let end = make_end_event(parent_uuid, None, "llm", ScopeType::Llm, None); processor.process(&start); processor.process(&chunk); + processor.process(&aliased_chunk); processor.process(&end); processor.force_flush().unwrap(); let spans = exporter.get_finished_spans().unwrap(); assert_eq!(spans.len(), 1); - assert_eq!(spans[0].events.events.len(), 1); + assert_eq!(spans[0].events.events.len(), 2); assert_eq!(spans[0].events.events[0].name.as_ref(), "llm.chunk"); + assert_eq!(spans[0].events.events[1].name.as_ref(), "hook_mark"); } #[test] diff --git a/crates/core/tests/unit/observability/otel_tests.rs b/crates/core/tests/unit/observability/otel_tests.rs index a31cbc956..4f0c7a07c 100644 --- a/crates/core/tests/unit/observability/otel_tests.rs +++ b/crates/core/tests/unit/observability/otel_tests.rs @@ -940,17 +940,29 @@ fn tool_projection_keeps_llm_chunks_as_parent_span_events() { let parent_uuid = Uuid::now_v7(); let start = make_start_event(parent_uuid, None, "llm", ScopeType::Llm, None); let chunk = make_mark_event(Some(parent_uuid), "llm.chunk", Some(json!({"delta": "x"}))); + let aliased_chunk = Event::Mark(MarkEvent::new( + BaseEvent::builder() + .parent_uuid(parent_uuid) + .name("hook_mark") + .metadata(json!({"hook_event_name": "llm.chunk"})) + .data(json!({"delta": "y"})) + .build(), + None, + None, + )); let end = make_end_event(parent_uuid, None, "llm", ScopeType::Llm, None); processor.process(&start); processor.process(&chunk); + processor.process(&aliased_chunk); processor.process(&end); processor.force_flush().unwrap(); let spans = exporter.get_finished_spans().unwrap(); assert_eq!(spans.len(), 1); - assert_eq!(spans[0].events.events.len(), 1); + assert_eq!(spans[0].events.events.len(), 2); assert_eq!(spans[0].events.events[0].name.as_ref(), "llm.chunk"); + assert_eq!(spans[0].events.events[1].name.as_ref(), "hook_mark"); } #[test] From b195834e363095a33e058d297d68687e4b0b314a Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 7 Jul 2026 17:23:08 -0600 Subject: [PATCH 07/14] feat(observability): add inherit mark projection mode Signed-off-by: Bryan Bednarski --- crates/core/src/observability/atif.rs | 13 ++++++++----- crates/core/src/observability/mod.rs | 4 +++- .../core/src/observability/plugin_component.rs | 10 +++++----- crates/core/tests/unit/atif_tests.rs | 18 ++++++++++++++++++ .../unit/observability/openinference_tests.rs | 2 +- .../tests/unit/observability/otel_tests.rs | 2 +- .../observability/plugin_component_tests.rs | 12 +++++++++--- crates/node/observability.d.ts | 4 ++-- crates/node/observability.js | 4 ++-- .../node/tests/observability_plugin_tests.mjs | 4 ++-- docs/configure-plugins/observability/atif.mdx | 5 +++-- .../observability/openinference.mdx | 4 ++-- .../observability/opentelemetry.mdx | 4 ++-- go/nemo_relay/observability_plugin.go | 6 ++++-- go/nemo_relay/observability_plugin_test.go | 4 ++-- python/nemo_relay/observability.py | 6 +++--- python/nemo_relay/observability.pyi | 2 +- python/tests/test_observability_plugin.py | 4 ++-- 18 files changed, 70 insertions(+), 38 deletions(-) diff --git a/crates/core/src/observability/atif.rs b/crates/core/src/observability/atif.rs index 216eb19bc..597b69b71 100644 --- a/crates/core/src/observability/atif.rs +++ b/crates/core/src/observability/atif.rs @@ -368,7 +368,8 @@ impl AtifExporter { /// Selects how point-in-time marks are represented in ATIF output. /// - /// The default [`MarkProjection::Event`] preserves marks as system steps. + /// The default [`MarkProjection::Inherit`] preserves exporter-native mark + /// handling. [`MarkProjection::Event`] forces system steps, while /// [`MarkProjection::Tool`] creates deterministic agent/tool steps for /// consumers that visualize tool calls but do not render system events. pub fn with_mark_projection(self, mark_projection: MarkProjection) -> Self { @@ -2633,13 +2634,15 @@ impl StepConversionState { } fn handle_mark(&mut self, mark: &Event, lookups: &EventLookupMaps) { - if is_llm_chunk_mark(mark) { + let excluded = mark_name_is_excluded(mark, &self.mark_exclude_names); + if (self.mark_projection == MarkProjection::Inherit + || (self.mark_projection == MarkProjection::Tool && excluded)) + && is_llm_chunk_mark(mark) + { return; } self.flush_observations(); - if self.mark_projection == MarkProjection::Tool - && !mark_name_is_excluded(mark, &self.mark_exclude_names) - { + if self.mark_projection == MarkProjection::Tool && !excluded { self.handle_mark_as_tool(mark, lookups, mark.data()); return; } diff --git a/crates/core/src/observability/mod.rs b/crates/core/src/observability/mod.rs index 724f8c142..b57d2bbb9 100644 --- a/crates/core/src/observability/mod.rs +++ b/crates/core/src/observability/mod.rs @@ -32,8 +32,10 @@ pub mod plugin_component; #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "snake_case")] pub enum MarkProjection { - /// Preserve marks as ATIF system steps or OTEL span events. + /// Use each exporter’s native handling for marks. #[default] + Inherit, + /// Force marks into ATIF system steps or OTEL span events. Event, /// Render non-excluded marks as deterministic ATIF tool steps or /// zero-duration OTEL child spans so trace-tree consumers can display them diff --git a/crates/core/src/observability/plugin_component.rs b/crates/core/src/observability/plugin_component.rs index d17b0ab61..9053d7585 100644 --- a/crates/core/src/observability/plugin_component.rs +++ b/crates/core/src/observability/plugin_component.rs @@ -228,7 +228,7 @@ pub struct AtifSectionConfig { /// Default model name. #[serde(default = "default_model_name")] pub model_name: String, - /// Representation used for mark events: `event` or `tool`. + /// Representation used for mark events: `inherit`, `event`, or `tool`. #[serde(default)] #[cfg_attr(feature = "schema", schemars(schema_with = "mark_projection_schema"))] pub mark_projection: MarkProjection, @@ -384,7 +384,7 @@ pub struct OtlpSectionConfig { /// Whether the subscriber is active. #[serde(default)] pub enabled: bool, - /// Representation used for mark events: `event` or `tool`. + /// Representation used for mark events: `inherit`, `event`, or `tool`. #[serde(default)] #[cfg_attr(feature = "schema", schemars(schema_with = "mark_projection_schema"))] pub mark_projection: MarkProjection, @@ -495,7 +495,7 @@ crate::editor_config! { agent_name => { label: "agent_name", kind: String }, agent_version => { label: "agent_version", kind: String }, model_name => { label: "model_name", kind: String }, - mark_projection => { label: "mark_projection", kind: Enum, values: ["event", "tool"] }, + mark_projection => { label: "mark_projection", kind: Enum, values: ["inherit", "event", "tool"] }, mark_exclude_names => { label: "mark_exclude_names", kind: Json }, tool_definitions => { label: "tool_definitions", kind: Json, optional: true }, extra => { label: "extra", kind: Json, optional: true }, @@ -508,7 +508,7 @@ crate::editor_config! { crate::editor_config! { impl OtlpSectionConfig { enabled => { label: "enabled", kind: Boolean }, - mark_projection => { label: "mark_projection", kind: Enum, values: ["event", "tool"] }, + mark_projection => { label: "mark_projection", kind: Enum, values: ["inherit", "event", "tool"] }, mark_exclude_names => { label: "mark_exclude_names", kind: Json }, transport => { label: "transport", kind: Enum, values: ["http_binary", "grpc"] }, endpoint => { label: "endpoint", kind: String, optional: true }, @@ -608,7 +608,7 @@ fn otlp_transport_schema( fn mark_projection_schema( generator: &mut schemars::r#gen::SchemaGenerator, ) -> schemars::schema::Schema { - string_enum_schema(generator, &["event", "tool"], Some("event")) + string_enum_schema(generator, &["inherit", "event", "tool"], Some("inherit")) } #[cfg(feature = "schema")] diff --git a/crates/core/tests/unit/atif_tests.rs b/crates/core/tests/unit/atif_tests.rs index 6c4f4e09b..763c3140d 100644 --- a/crates/core/tests/unit/atif_tests.rs +++ b/crates/core/tests/unit/atif_tests.rs @@ -3623,6 +3623,24 @@ fn test_exporter_skips_llm_chunk_marks() { assert_eq!(trajectory.steps[0].message, json!("agent.status")); } +#[test] +fn test_exporter_event_projection_includes_llm_chunk_marks() { + let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()) + .with_mark_projection(MarkProjection::Event); + + exporter.state.lock().unwrap().events.push( + event_builder(Uuid::now_v7(), EventType::Mark) + .name("llm.chunk") + .data(json!({"delta": "partial"})) + .build(), + ); + + let trajectory = exporter.export().unwrap(); + assert_eq!(trajectory.steps.len(), 1); + assert_eq!(trajectory.steps[0].source, "system"); + assert_eq!(trajectory.steps[0].message, json!("llm.chunk")); +} + #[test] fn test_exporter_dedupes_overlapping_hook_and_gateway_llm_spans() { let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); diff --git a/crates/core/tests/unit/observability/openinference_tests.rs b/crates/core/tests/unit/observability/openinference_tests.rs index ca2b61311..77fdb17dc 100644 --- a/crates/core/tests/unit/observability/openinference_tests.rs +++ b/crates/core/tests/unit/observability/openinference_tests.rs @@ -495,7 +495,7 @@ fn config_defaults_and_builder_overrides_are_applied() { assert_eq!(defaults.transport, OtlpTransport::HttpBinary); assert_eq!(defaults.service_name, "nemo-relay"); assert_eq!(defaults.instrumentation_scope, "nemo-relay-openinference"); - assert_eq!(defaults.mark_projection, MarkProjection::Event); + assert_eq!(defaults.mark_projection, MarkProjection::Inherit); assert_eq!(defaults.mark_exclude_names, vec!["llm.chunk"]); assert_eq!(defaults.timeout, Duration::from_secs(3)); assert!(defaults.headers.is_empty()); diff --git a/crates/core/tests/unit/observability/otel_tests.rs b/crates/core/tests/unit/observability/otel_tests.rs index 4f0c7a07c..ff164d527 100644 --- a/crates/core/tests/unit/observability/otel_tests.rs +++ b/crates/core/tests/unit/observability/otel_tests.rs @@ -403,7 +403,7 @@ fn config_defaults_and_builder_overrides_are_applied() { assert_eq!(defaults.transport, OtlpTransport::HttpBinary); assert_eq!(defaults.service_name, "nemo-relay"); assert_eq!(defaults.instrumentation_scope, "nemo-relay-otel"); - assert_eq!(defaults.mark_projection, MarkProjection::Event); + assert_eq!(defaults.mark_projection, MarkProjection::Inherit); assert_eq!(defaults.mark_exclude_names, vec!["llm.chunk"]); assert_eq!(defaults.timeout, Duration::from_secs(3)); assert!(defaults.headers.is_empty()); diff --git a/crates/core/tests/unit/observability/plugin_component_tests.rs b/crates/core/tests/unit/observability/plugin_component_tests.rs index b09031b4d..8d9b2652a 100644 --- a/crates/core/tests/unit/observability/plugin_component_tests.rs +++ b/crates/core/tests/unit/observability/plugin_component_tests.rs @@ -216,13 +216,13 @@ fn default_config_and_component_conversion_cover_public_shape() { assert_eq!(atif.agent_name, "NeMo Relay"); assert_eq!(atif.agent_version, env!("CARGO_PKG_VERSION")); assert_eq!(atif.model_name, "unknown"); - assert_eq!(atif.mark_projection, MarkProjection::Event); + assert_eq!(atif.mark_projection, MarkProjection::Inherit); assert_eq!(atif.mark_exclude_names, vec!["llm.chunk"]); assert_eq!(atif.filename_template, "nemo-relay-atif-{session_id}.json"); let otlp = OtlpSectionConfig::default(); assert!(!otlp.enabled); - assert_eq!(otlp.mark_projection, MarkProjection::Event); + assert_eq!(otlp.mark_projection, MarkProjection::Inherit); assert_eq!(otlp.mark_exclude_names, vec!["llm.chunk"]); assert_eq!(otlp.transport, "http_binary"); assert_eq!(otlp.service_name, "nemo-relay"); @@ -255,6 +255,12 @@ fn mark_projection_parses_per_exporter_and_rejects_unknown_values() { assert_eq!(atif.mark_projection, MarkProjection::Tool); assert_eq!(otlp.mark_projection, MarkProjection::Tool); + let inherited: AtifSectionConfig = serde_json::from_value(json!({ + "mark_projection": "inherit" + })) + .unwrap(); + assert_eq!(inherited.mark_projection, MarkProjection::Inherit); + let custom_exclusions: OtlpSectionConfig = serde_json::from_value(json!({ "mark_projection": "tool", "mark_exclude_names": ["notification", "hook_mark"] @@ -336,7 +342,7 @@ fn schema_contains_every_supported_observability_option() { assert!(schema_property_has_enum( &schema, "mark_projection", - &["event", "tool"] + &["inherit", "event", "tool"] )); assert!(schema_property_has_default( &schema, diff --git a/crates/node/observability.d.ts b/crates/node/observability.d.ts index 70ea82f82..53d3319cc 100644 --- a/crates/node/observability.d.ts +++ b/crates/node/observability.d.ts @@ -47,7 +47,7 @@ export interface AtifConfig { agent_name?: string; agent_version?: string; model_name?: string; - mark_projection?: 'event' | 'tool'; + mark_projection?: 'inherit' | 'event' | 'tool'; mark_exclude_names?: string[]; tool_definitions?: Record[]; extra?: Record; @@ -58,7 +58,7 @@ export interface AtifConfig { export interface OtlpConfig { enabled?: boolean; - mark_projection?: 'event' | 'tool'; + mark_projection?: 'inherit' | 'event' | 'tool'; mark_exclude_names?: string[]; transport?: 'http_binary' | 'grpc' | string; endpoint?: string; diff --git a/crates/node/observability.js b/crates/node/observability.js index 47a875531..05f7e72b6 100644 --- a/crates/node/observability.js +++ b/crates/node/observability.js @@ -43,7 +43,7 @@ function atifConfig(config = {}) { enabled: false, agent_name: 'NeMo Relay', model_name: 'unknown', - mark_projection: 'event', + mark_projection: 'inherit', mark_exclude_names: ['llm.chunk'], filename_template: 'nemo-relay-atif-{session_id}.json', ...config, @@ -59,7 +59,7 @@ function atifConfig(config = {}) { function otlpConfig(config = {}) { return { enabled: false, - mark_projection: 'event', + mark_projection: 'inherit', mark_exclude_names: ['llm.chunk'], transport: 'http_binary', headers: {}, diff --git a/crates/node/tests/observability_plugin_tests.mjs b/crates/node/tests/observability_plugin_tests.mjs index d09b9c3cf..c384ebe33 100644 --- a/crates/node/tests/observability_plugin_tests.mjs +++ b/crates/node/tests/observability_plugin_tests.mjs @@ -25,13 +25,13 @@ describe('observability plugin helpers', () => { enabled: false, agent_name: 'NeMo Relay', model_name: 'unknown', - mark_projection: 'event', + mark_projection: 'inherit', mark_exclude_names: ['llm.chunk'], filename_template: 'nemo-relay-atif-{session_id}.json', }); assert.deepEqual(observability.otlpConfig(), { enabled: false, - mark_projection: 'event', + mark_projection: 'inherit', mark_exclude_names: ['llm.chunk'], transport: 'http_binary', headers: {}, diff --git a/docs/configure-plugins/observability/atif.mdx b/docs/configure-plugins/observability/atif.mdx index 9af2ad413..3e8b19a01 100644 --- a/docs/configure-plugins/observability/atif.mdx +++ b/docs/configure-plugins/observability/atif.mdx @@ -53,7 +53,7 @@ The following table describes the top-level ATIF settings: | `agent_name` | `NeMo Relay` | Agent metadata written into the trajectory. | | `agent_version` | NeMo Relay crate version | Agent version metadata. | | `model_name` | `unknown` | Default model metadata when no call-level model is present. | -| `mark_projection` | `event` | `event` preserves marks as system steps. `tool` emits deterministic agent/tool steps for trajectory viewers that render tool calls but not system events. | +| `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces system steps; `tool` emits deterministic agent/tool steps for trajectory viewers that render tool calls but not system events. | | `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain native event handling. Metadata `hook_event_name` aliases are also matched. | | `tool_definitions` | Omitted | Optional ATIF tool metadata. | | `extra` | Omitted | Optional ATIF agent metadata. | @@ -196,7 +196,8 @@ per-destination sink failures to the failed sink. ## Expected Output -By default, marks remain point-in-time `system` steps with their payload, +By default, marks use exporter-native handling. In ATIF, ordinary marks remain +point-in-time `system` steps with their payload, category, category profile, timestamp, and ancestry preserved in `extra`. Set `mark_projection = "tool"` when the target viewer requires a visible tool-call projection. The projected step uses `llm_call_count = 0`, retains the original diff --git a/docs/configure-plugins/observability/openinference.mdx b/docs/configure-plugins/observability/openinference.mdx index 1efa3ad9c..3e76fd547 100644 --- a/docs/configure-plugins/observability/openinference.mdx +++ b/docs/configure-plugins/observability/openinference.mdx @@ -61,7 +61,7 @@ OpenInference uses the same OTLP section shape as | Field | Default | Notes | |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | -| `mark_projection` | `event` | `event` attaches marks to active parent spans. `tool` emits zero-duration OpenInference `TOOL` child spans. | +| `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces span events; `tool` emits zero-duration OpenInference `TOOL` child spans. | | `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain native event handling. Metadata `hook_event_name` aliases are also matched. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | @@ -85,7 +85,7 @@ and invocation parameters are emitted when typed codec annotations supply them. Exported LLM attributes exclude request headers and other non-observable transport metadata. -The default `event` projection preserves marks as span events. With +The default `inherit` projection preserves exporter-native mark handling. With `mark_projection = "tool"`, each non-streaming mark becomes a zero-duration child span with `openinference.span.kind = "TOOL"` and the original mark payload, category, profile, timestamp, and parent identifiers retained as attributes. High-volume diff --git a/docs/configure-plugins/observability/opentelemetry.mdx b/docs/configure-plugins/observability/opentelemetry.mdx index 95e23088f..4cccc69aa 100644 --- a/docs/configure-plugins/observability/opentelemetry.mdx +++ b/docs/configure-plugins/observability/opentelemetry.mdx @@ -55,7 +55,7 @@ The following table describes OpenTelemetry exporter settings: | Field | Default | Notes | |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | -| `mark_projection` | `event` | `event` attaches marks to active parent spans. `tool` emits zero-duration child spans for trace-tree visibility. | +| `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces span events; `tool` emits zero-duration child spans for trace-tree visibility. | | `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain native event handling. Metadata `hook_event_name` aliases are also matched. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | @@ -73,7 +73,7 @@ The collector should receive OTLP trace export requests. The tracing backend should show spans for NeMo Relay scopes, tools, LLM calls, and marks grouped by root scope. -The default `event` projection follows the OpenTelemetry span-event model. Set +The default `inherit` projection follows the OpenTelemetry span-event model. Set `mark_projection = "tool"` only when the target backend needs non-streaming marks represented as visible child spans. Projected marks use `SpanKind::Internal`, carry `nemo_relay.mark.projection = "tool"`, and retain mark UUID, parentage, diff --git a/go/nemo_relay/observability_plugin.go b/go/nemo_relay/observability_plugin.go index 2e481e62c..c95af33c4 100644 --- a/go/nemo_relay/observability_plugin.go +++ b/go/nemo_relay/observability_plugin.go @@ -12,6 +12,8 @@ const ObservabilityPluginKind = "observability" type ObservabilityMarkProjection string const ( + // ObservabilityMarkProjectionInherit preserves exporter-native mark handling. + ObservabilityMarkProjectionInherit ObservabilityMarkProjection = "inherit" // ObservabilityMarkProjectionEvent preserves marks as events. ObservabilityMarkProjectionEvent ObservabilityMarkProjection = "event" // ObservabilityMarkProjectionTool emits visible tool projections. @@ -177,7 +179,7 @@ func NewObservabilityAtifConfig() ObservabilityAtifConfig { return ObservabilityAtifConfig{ AgentName: "NeMo Relay", ModelName: "unknown", - MarkProjection: ObservabilityMarkProjectionEvent, + MarkProjection: ObservabilityMarkProjectionInherit, MarkExcludeNames: []string{"llm.chunk"}, FilenameTemplate: "nemo-relay-atif-{session_id}.json", } @@ -197,7 +199,7 @@ func NewObservabilityHttpStorageConfig(endpoint string) ObservabilityHttpStorage func NewObservabilityOtlpConfig() ObservabilityOtlpConfig { return ObservabilityOtlpConfig{ Transport: "http_binary", - MarkProjection: ObservabilityMarkProjectionEvent, + MarkProjection: ObservabilityMarkProjectionInherit, MarkExcludeNames: []string{"llm.chunk"}, Headers: map[string]string{}, ResourceAttributes: map[string]string{}, diff --git a/go/nemo_relay/observability_plugin_test.go b/go/nemo_relay/observability_plugin_test.go index 62533dc6f..399d899f4 100644 --- a/go/nemo_relay/observability_plugin_test.go +++ b/go/nemo_relay/observability_plugin_test.go @@ -43,7 +43,7 @@ func TestObservabilityConfigHelpers(t *testing.T) { FieldNamePolicy: "replace_dots", }} atif := NewObservabilityAtifConfig() - if atif.Enabled || atif.AgentName != "NeMo Relay" || atif.ModelName != "unknown" || atif.MarkProjection != ObservabilityMarkProjectionEvent || len(atif.MarkExcludeNames) != 1 || atif.MarkExcludeNames[0] != "llm.chunk" || atif.FilenameTemplate != "nemo-relay-atif-{session_id}.json" { + if atif.Enabled || atif.AgentName != "NeMo Relay" || atif.ModelName != "unknown" || atif.MarkProjection != ObservabilityMarkProjectionInherit || len(atif.MarkExcludeNames) != 1 || atif.MarkExcludeNames[0] != "llm.chunk" || atif.FilenameTemplate != "nemo-relay-atif-{session_id}.json" { t.Fatalf("unexpected ATIF defaults: %#v", atif) } allowHTTP := false @@ -64,7 +64,7 @@ func TestObservabilityConfigHelpers(t *testing.T) { httpStorage, } otlp := NewObservabilityOtlpConfig() - if otlp.Enabled || otlp.MarkProjection != ObservabilityMarkProjectionEvent || len(otlp.MarkExcludeNames) != 1 || otlp.MarkExcludeNames[0] != "llm.chunk" || otlp.Transport != "http_binary" || otlp.ServiceName != "nemo-relay" || otlp.TimeoutMillis != 3000 { + if otlp.Enabled || otlp.MarkProjection != ObservabilityMarkProjectionInherit || len(otlp.MarkExcludeNames) != 1 || otlp.MarkExcludeNames[0] != "llm.chunk" || otlp.Transport != "http_binary" || otlp.ServiceName != "nemo-relay" || otlp.TimeoutMillis != 3000 { t.Fatalf("unexpected OTLP defaults: %#v", otlp) } atif.MarkProjection = ObservabilityMarkProjectionTool diff --git a/python/nemo_relay/observability.py b/python/nemo_relay/observability.py index f565a491f..541b74be1 100644 --- a/python/nemo_relay/observability.py +++ b/python/nemo_relay/observability.py @@ -10,7 +10,7 @@ from nemo_relay import Json, JsonObject, UnsupportedBehavior -MarkProjection = Literal["event", "tool"] +MarkProjection = Literal["inherit", "event", "tool"] class _SupportsToDict(Protocol): @@ -167,7 +167,7 @@ class AtifConfig: agent_name: str = "NeMo Relay" agent_version: str | None = None model_name: str = "unknown" - mark_projection: MarkProjection = "event" + mark_projection: MarkProjection = "inherit" mark_exclude_names: list[str] = field(default_factory=lambda: ["llm.chunk"]) tool_definitions: list[JsonObject] | None = None extra: JsonObject | None = None @@ -200,7 +200,7 @@ class OtlpConfig: """Shared OpenTelemetry/OpenInference OTLP export settings.""" enabled: bool = False - mark_projection: MarkProjection = "event" + mark_projection: MarkProjection = "inherit" mark_exclude_names: list[str] = field(default_factory=lambda: ["llm.chunk"]) transport: Literal["http_binary", "grpc"] = "http_binary" endpoint: str | None = None diff --git a/python/nemo_relay/observability.pyi b/python/nemo_relay/observability.pyi index 61c5af8af..a12b99852 100644 --- a/python/nemo_relay/observability.pyi +++ b/python/nemo_relay/observability.pyi @@ -10,7 +10,7 @@ from typing import Literal from nemo_relay import JsonObject, UnsupportedBehavior -MarkProjection = Literal["event", "tool"] +MarkProjection = Literal["inherit", "event", "tool"] @dataclass(slots=True) class ConfigPolicy: diff --git a/python/tests/test_observability_plugin.py b/python/tests/test_observability_plugin.py index 60378ce76..dcd1774be 100644 --- a/python/tests/test_observability_plugin.py +++ b/python/tests/test_observability_plugin.py @@ -34,13 +34,13 @@ def test_defaults_and_component_wrapper(self): "enabled": False, "agent_name": "NeMo Relay", "model_name": "unknown", - "mark_projection": "event", + "mark_projection": "inherit", "mark_exclude_names": ["llm.chunk"], "filename_template": "nemo-relay-atif-{session_id}.json", } assert OtlpConfig().to_dict() == { "enabled": False, - "mark_projection": "event", + "mark_projection": "inherit", "mark_exclude_names": ["llm.chunk"], "transport": "http_binary", "headers": {}, From 39f42fb85b1e143eb60e426f0e74cf05fedeea6d Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 7 Jul 2026 17:42:47 -0600 Subject: [PATCH 08/14] docs(observability): clarify mark projection modes Signed-off-by: Bryan Bednarski --- docs/configure-plugins/observability/atif.mdx | 14 ++++++++------ .../observability/openinference.mdx | 18 +++++++++++------- .../observability/opentelemetry.mdx | 17 ++++++++++------- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/docs/configure-plugins/observability/atif.mdx b/docs/configure-plugins/observability/atif.mdx index 3e8b19a01..bd8d3abdd 100644 --- a/docs/configure-plugins/observability/atif.mdx +++ b/docs/configure-plugins/observability/atif.mdx @@ -54,7 +54,7 @@ The following table describes the top-level ATIF settings: | `agent_version` | NeMo Relay crate version | Agent version metadata. | | `model_name` | `unknown` | Default model metadata when no call-level model is present. | | `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces system steps; `tool` emits deterministic agent/tool steps for trajectory viewers that render tool calls but not system events. | -| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain native event handling. Metadata `hook_event_name` aliases are also matched. | +| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain exporter-native handling. Metadata `hook_event_name` aliases are also matched. | | `tool_definitions` | Omitted | Optional ATIF tool metadata. | | `extra` | Omitted | Optional ATIF agent metadata. | | `output_directory` | Current working directory | Directory containing trajectory files. Ignored when `storage` is non-empty. | @@ -196,9 +196,9 @@ per-destination sink failures to the failed sink. ## Expected Output -By default, marks use exporter-native handling. In ATIF, ordinary marks remain -point-in-time `system` steps with their payload, -category, category profile, timestamp, and ancestry preserved in `extra`. Set +By default, marks use exporter-native handling. In ATIF, marks with payloads +remain point-in-time `system` steps with their payload, category, category +profile, timestamp, and ancestry preserved in `extra`. Set `mark_projection = "tool"` when the target viewer requires a visible tool-call projection. The projected step uses `llm_call_count = 0`, retains the original mark data in `extra`, and does not change the canonical ATOF event. @@ -206,8 +206,10 @@ mark data in `extra`, and does not change the canonical ATOF event. Add other event names to `mark_exclude_names` when a trajectory should keep those marks in their native system-step representation rather than projecting them as tool steps. The exclusion list affects only tool projection; it does -not remove mark payload or metadata. ATIF continues to suppress the built-in -high-volume `llm.chunk` stream receipts as it does in event mode. +not remove mark payload or metadata. ATIF suppresses the built-in high-volume +`llm.chunk` stream receipts in `inherit` mode and when they are excluded from +`tool` projection. `event` explicitly includes payload-bearing `llm.chunk` +marks as system steps. The exporter translates NeMo Relay lifecycle events into ATIF v1.7 trajectory data. LLM start and end events become model steps, tool events become tool diff --git a/docs/configure-plugins/observability/openinference.mdx b/docs/configure-plugins/observability/openinference.mdx index 3e76fd547..4eb051ce3 100644 --- a/docs/configure-plugins/observability/openinference.mdx +++ b/docs/configure-plugins/observability/openinference.mdx @@ -62,7 +62,7 @@ OpenInference uses the same OTLP section shape as |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | | `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces span events; `tool` emits zero-duration OpenInference `TOOL` child spans. | -| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain native event handling. Metadata `hook_event_name` aliases are also matched. | +| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain exporter-native handling. Metadata `hook_event_name` aliases are also matched. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | | `headers` | `{}` | String-to-string exporter headers. | @@ -85,14 +85,18 @@ and invocation parameters are emitted when typed codec annotations supply them. Exported LLM attributes exclude request headers and other non-observable transport metadata. -The default `inherit` projection preserves exporter-native mark handling. With -`mark_projection = "tool"`, each non-streaming mark becomes a zero-duration child span with -`openinference.span.kind = "TOOL"` and the original mark payload, category, -profile, timestamp, and parent identifiers retained as attributes. High-volume -`llm.chunk` marks remain span events. +The default `inherit` projection preserves exporter-native mark handling: a +mark with an active parent span is a span event, while an orphan mark is a +standalone zero-duration mark span. `mark_projection = "event"` explicitly +selects that representation. With `mark_projection = "tool"`, each +non-streaming mark becomes a zero-duration span (a child span when a parent is +available) with `openinference.span.kind = "TOOL"` and the original mark +payload, category, profile, timestamp, and parent identifiers retained as +attributes. High-volume `llm.chunk` marks retain exporter-native handling by +default and when excluded from tool projection. Add other event names to `mark_exclude_names` when a backend should keep those -marks as native span events rather than visible tool children. The exclusion +marks in exporter-native form rather than visible tool children. The exclusion list affects only tool projection; it does not remove mark payload or metadata. Each lifecycle span includes `nemo_relay.uuid` and `nemo_relay.parent_uuid` diff --git a/docs/configure-plugins/observability/opentelemetry.mdx b/docs/configure-plugins/observability/opentelemetry.mdx index 4cccc69aa..6f6e22c26 100644 --- a/docs/configure-plugins/observability/opentelemetry.mdx +++ b/docs/configure-plugins/observability/opentelemetry.mdx @@ -56,7 +56,7 @@ The following table describes OpenTelemetry exporter settings: |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | | `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces span events; `tool` emits zero-duration child spans for trace-tree visibility. | -| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain native event handling. Metadata `hook_event_name` aliases are also matched. | +| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain exporter-native handling. Metadata `hook_event_name` aliases are also matched. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | | `headers` | `{}` | String-to-string exporter headers. | @@ -73,15 +73,18 @@ The collector should receive OTLP trace export requests. The tracing backend should show spans for NeMo Relay scopes, tools, LLM calls, and marks grouped by root scope. -The default `inherit` projection follows the OpenTelemetry span-event model. Set -`mark_projection = "tool"` only when the target backend needs non-streaming -marks represented as visible child spans. Projected marks use `SpanKind::Internal`, carry +The default `inherit` projection follows exporter-native handling: a mark with +an active parent span is a span event, while an orphan mark is a standalone +zero-duration `mark:` span. `mark_projection = "event"` explicitly selects +that representation. Set `mark_projection = "tool"` only when the target +backend needs non-streaming marks represented as visible child spans. Projected +marks use `SpanKind::Internal`, carry `nemo_relay.mark.projection = "tool"`, and retain mark UUID, parentage, -category/profile, and payload attributes. High-volume `llm.chunk` marks remain -span events. +category/profile, and payload attributes. High-volume `llm.chunk` marks retain +exporter-native handling by default and when excluded from tool projection. Add other event names to `mark_exclude_names` when a backend should keep those -marks as native span events rather than visible tool children. The exclusion +marks in exporter-native form rather than visible tool children. The exclusion list affects only tool projection; it does not remove mark payload or metadata. Each lifecycle span includes `nemo_relay.uuid` and `nemo_relay.parent_uuid` From 2eec4a722d4b98420267131c65148393d1fe8594 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 7 Jul 2026 17:51:38 -0600 Subject: [PATCH 09/14] refactor(observability): share mark projection policy Signed-off-by: Bryan Bednarski --- crates/core/src/observability/atif.rs | 15 +++++++-------- crates/core/src/observability/mod.rs | 16 ++++++++++++++++ crates/core/src/observability/openinference.rs | 10 +++++----- crates/core/src/observability/otel.rs | 8 ++++---- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/crates/core/src/observability/atif.rs b/crates/core/src/observability/atif.rs index 597b69b71..cfe072598 100644 --- a/crates/core/src/observability/atif.rs +++ b/crates/core/src/observability/atif.rs @@ -44,8 +44,9 @@ use crate::error::Result; use crate::json::Json; use super::{ - MarkProjection, default_mark_exclude_names, estimate_cost_for_response_or_model, - is_llm_chunk_mark, manual, mark_name_is_excluded, merge_usage, model_name_for_llm_event, + MarkProjection, default_mark_exclude_names, effective_mark_projection, + estimate_cost_for_response_or_model, is_llm_chunk_mark, manual, merge_usage, + model_name_for_llm_event, }; /// The ATIF schema version string embedded in all exported trajectories. @@ -2634,15 +2635,13 @@ impl StepConversionState { } fn handle_mark(&mut self, mark: &Event, lookups: &EventLookupMaps) { - let excluded = mark_name_is_excluded(mark, &self.mark_exclude_names); - if (self.mark_projection == MarkProjection::Inherit - || (self.mark_projection == MarkProjection::Tool && excluded)) - && is_llm_chunk_mark(mark) - { + let projection = + effective_mark_projection(mark, self.mark_projection, &self.mark_exclude_names); + if projection == MarkProjection::Inherit && is_llm_chunk_mark(mark) { return; } self.flush_observations(); - if self.mark_projection == MarkProjection::Tool && !excluded { + if projection == MarkProjection::Tool { self.handle_mark_as_tool(mark, lookups, mark.data()); return; } diff --git a/crates/core/src/observability/mod.rs b/crates/core/src/observability/mod.rs index b57d2bbb9..f202b10c3 100644 --- a/crates/core/src/observability/mod.rs +++ b/crates/core/src/observability/mod.rs @@ -68,6 +68,22 @@ pub(crate) fn mark_name_is_excluded( }) } +/// Resolves a configured mark projection for one event. +/// +/// Exclusions only affect tool projection; all other modes retain their +/// configured exporter-native behavior. +pub(crate) fn effective_mark_projection( + event: &crate::api::event::Event, + projection: MarkProjection, + excluded_names: &[String], +) -> MarkProjection { + if projection == MarkProjection::Tool && mark_name_is_excluded(event, excluded_names) { + MarkProjection::Inherit + } else { + projection + } +} + pub(crate) fn is_llm_chunk_mark(event: &crate::api::event::Event) -> bool { event.name() == "llm.chunk" || event diff --git a/crates/core/src/observability/openinference.rs b/crates/core/src/observability/openinference.rs index 1f9867207..15676277d 100644 --- a/crates/core/src/observability/openinference.rs +++ b/crates/core/src/observability/openinference.rs @@ -21,9 +21,9 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use super::{ - MarkProjection, default_mark_exclude_names, estimate_cost_for_response_or_model, - estimate_cost_for_response_or_requested_model, manual, mark_name_is_excluded, merge_usage, - model_name_for_llm_event, + MarkProjection, default_mark_exclude_names, effective_mark_projection, + estimate_cost_for_response_or_model, estimate_cost_for_response_or_requested_model, manual, + merge_usage, model_name_for_llm_event, }; use crate::api::event::{Event, EventNormalizationExt, ScopeCategory}; use crate::api::runtime::EventSubscriberFn; @@ -539,8 +539,8 @@ impl OpenInferenceEventProcessor { } fn process_mark(&mut self, event: &Event) { - if self.mark_projection == MarkProjection::Tool - && !mark_name_is_excluded(event, &self.mark_exclude_names) + if effective_mark_projection(event, self.mark_projection, &self.mark_exclude_names) + == MarkProjection::Tool { self.process_mark_as_tool(event); return; diff --git a/crates/core/src/observability/otel.rs b/crates/core/src/observability/otel.rs index ce277db3c..727b60a79 100644 --- a/crates/core/src/observability/otel.rs +++ b/crates/core/src/observability/otel.rs @@ -21,8 +21,8 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use super::{ - MarkProjection, default_mark_exclude_names, estimate_cost_for_response_or_model, - estimate_cost_for_response_or_requested_model, manual, mark_name_is_excluded, + MarkProjection, default_mark_exclude_names, effective_mark_projection, + estimate_cost_for_response_or_model, estimate_cost_for_response_or_requested_model, manual, model_name_for_llm_event, }; use crate::api::event::{Event, EventNormalizationExt, ScopeCategory}; @@ -534,8 +534,8 @@ impl OtelEventProcessor { } fn process_mark(&mut self, event: &Event) { - if self.mark_projection == MarkProjection::Tool - && !mark_name_is_excluded(event, &self.mark_exclude_names) + if effective_mark_projection(event, self.mark_projection, &self.mark_exclude_names) + == MarkProjection::Tool { self.process_mark_as_tool(event); return; From 18cd30f0a52ba009917fd9863e41b28a3d777350 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 8 Jul 2026 10:26:28 -0600 Subject: [PATCH 10/14] refactor(observability): keep ATIF step-oriented Signed-off-by: Bryan Bednarski --- crates/core/src/observability/atif.rs | 281 +-------------- crates/core/src/observability/mod.rs | 20 +- .../src/observability/plugin_component.rs | 17 +- .../tests/integration/middleware_tests.rs | 43 +-- crates/core/tests/unit/atif_tests.rs | 327 ++---------------- .../unit/observability/openinference_tests.rs | 4 +- .../tests/unit/observability/otel_tests.rs | 4 +- .../observability/plugin_component_tests.rs | 28 +- crates/node/observability.d.ts | 2 - crates/node/observability.js | 2 - .../node/tests/observability_plugin_tests.mjs | 3 - docs/configure-plugins/observability/atif.mdx | 22 +- docs/resources/glossary.mdx | 2 +- go/nemo_relay/observability_plugin.go | 4 - go/nemo_relay/observability_plugin_test.go | 5 +- python/nemo_relay/observability.py | 4 - python/nemo_relay/observability.pyi | 2 - python/tests/test_observability_plugin.py | 3 - 18 files changed, 88 insertions(+), 685 deletions(-) diff --git a/crates/core/src/observability/atif.rs b/crates/core/src/observability/atif.rs index cfe072598..6e1967260 100644 --- a/crates/core/src/observability/atif.rs +++ b/crates/core/src/observability/atif.rs @@ -22,7 +22,7 @@ //! | LLM End | `agent` step | Response content, tool_calls promoted| //! | Tool Start | *(skipped)* | tool_calls come from LLM End instead | //! | Tool End | agent observation | Correlated by `source_call_id` | -//! | Mark (with data)| `system` step | Optional deterministic tool projection | +//! | Mark | *(skipped)* | Point-in-time telemetry is not a step| //! | Scope Start/End | *(skipped)* | Structural events, not trajectory | //! //! The exporter serializes the full collected event stream into a single ATIF @@ -43,11 +43,7 @@ use crate::codec::response::AnnotatedLlmResponse; use crate::error::Result; use crate::json::Json; -use super::{ - MarkProjection, default_mark_exclude_names, effective_mark_projection, - estimate_cost_for_response_or_model, is_llm_chunk_mark, manual, merge_usage, - model_name_for_llm_event, -}; +use super::{estimate_cost_for_response_or_model, manual, merge_usage, model_name_for_llm_event}; /// The ATIF schema version string embedded in all exported trajectories. /// @@ -277,15 +273,6 @@ pub struct AtifStepExtra { /// Full raw LLM response payload for response-level fidelity. #[serde(skip_serializing_if = "Option::is_none")] pub llm_response: Option, - /// Full raw point-in-time event payload for mark/system steps. - #[serde(skip_serializing_if = "Option::is_none")] - pub event_payload: Option, - /// Canonical ATOF category for mark/system steps. - #[serde(skip_serializing_if = "Option::is_none")] - pub event_category: Option, - /// Canonical ATOF category profile for mark/system steps. - #[serde(skip_serializing_if = "Option::is_none")] - pub event_category_profile: Option, /// Per-tool callable lineage, aligned with `tool_calls`. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub tool_ancestry: Vec, @@ -332,8 +319,6 @@ pub struct AtifTrajectory { struct AtifExporterState { session_id: String, agent_info: AtifAgentInfo, - mark_projection: MarkProjection, - mark_exclude_names: Vec, events: Vec, } @@ -360,47 +345,25 @@ impl AtifExporter { state: Arc::new(Mutex::new(AtifExporterState { session_id, agent_info, - mark_projection: MarkProjection::default(), - mark_exclude_names: default_mark_exclude_names(), events: Vec::new(), })), } } - /// Selects how point-in-time marks are represented in ATIF output. - /// - /// The default [`MarkProjection::Inherit`] preserves exporter-native mark - /// handling. [`MarkProjection::Event`] forces system steps, while - /// [`MarkProjection::Tool`] creates deterministic agent/tool steps for - /// consumers that visualize tool calls but do not render system events. - pub fn with_mark_projection(self, mark_projection: MarkProjection) -> Self { - self.state.lock().unwrap().mark_projection = mark_projection; - self - } - - /// Excludes named marks from tool projection while preserving their native - /// event representation. The default excludes high-volume `llm.chunk` - /// marks. - pub fn with_mark_exclude_names(self, names: I) -> Self - where - I: IntoIterator, - S: Into, - { - self.state.lock().unwrap().mark_exclude_names = names.into_iter().map(Into::into).collect(); - self - } - /// Return an event subscriber function that records NeMo Relay events. /// /// The returned callback can be registered with /// [`register_subscriber`](crate::api::subscriber::register_subscriber). /// /// # Returns - /// An [`EventSubscriberFn`] that appends each observed event to this - /// exporter's internal buffer. + /// An [`EventSubscriberFn`] that appends compatible lifecycle events to + /// this exporter's internal buffer. Point-in-time marks are ignored. pub fn subscriber(&self) -> EventSubscriberFn { let state = self.state.clone(); Arc::new(move |event: &Event| { + if event.kind() == "mark" { + return; + } if let Ok(mut s) = state.lock() { s.events.push(event.clone()); } @@ -429,13 +392,11 @@ impl AtifExporter { /// callers that prefer an explicitly fallible method name. pub fn try_export(&self) -> Result { flush_subscribers()?; - let (session_id, agent_info, mark_projection, mark_exclude_names, events) = { + let (session_id, agent_info, events) = { let state = self.state.lock().unwrap(); ( state.session_id.clone(), state.agent_info.clone(), - state.mark_projection, - state.mark_exclude_names.clone(), state.events.clone(), ) }; @@ -443,8 +404,6 @@ impl AtifExporter { Ok(events_to_trajectory( &session_id, agent_info, - mark_projection, - &mark_exclude_names, &collected_events, )) } @@ -1928,9 +1887,6 @@ impl PendingAgentStep { invocation: self.invocation.take(), llm_request: None, llm_response: self.llm_response.take(), - event_payload: None, - event_category: None, - event_category_profile: None, tool_ancestry: std::mem::take(&mut self.tool_ancestry), tool_invocations: if self.tool_invocations.is_empty() { None @@ -2008,8 +1964,6 @@ impl PendingAgentStep { #[derive(Default)] struct StepConversionState { - mark_projection: MarkProjection, - mark_exclude_names: Vec, steps: Vec, last_tool_call_map: std::collections::HashMap, tool_scope_call_ids: std::collections::HashMap, @@ -2049,7 +2003,6 @@ impl StepConversionState { ("scope", Some(crate::api::event::ScopeCategory::End), Some("tool")) => { self.handle_tool_end(event, lookups) } - ("mark", _, _) => self.handle_mark(event, lookups), _ => {} } } @@ -2288,9 +2241,6 @@ impl StepConversionState { invocation: None, llm_request: Some(content.clone()), llm_response: None, - event_payload: None, - event_category: None, - event_category_profile: None, tool_ancestry: Vec::new(), tool_invocations: None, }; @@ -2634,131 +2584,6 @@ impl StepConversionState { true } - fn handle_mark(&mut self, mark: &Event, lookups: &EventLookupMaps) { - let projection = - effective_mark_projection(mark, self.mark_projection, &self.mark_exclude_names); - if projection == MarkProjection::Inherit && is_llm_chunk_mark(mark) { - return; - } - self.flush_observations(); - if projection == MarkProjection::Tool { - self.handle_mark_as_tool(mark, lookups, mark.data()); - return; - } - let Some(data) = mark.data() else { - return; - }; - if is_empty_mark_payload(data) { - return; - } - let extra = AtifStepExtra { - ancestry: build_ancestry(mark, &lookups.name_map), - invocation: Some(AtifInvocationInfo { - start_timestamp: None, - end_timestamp: None, - invocation_id: Some(mark.uuid().to_string()), - status: Some("completed".to_string()), - framework: Some("nemo_relay".to_string()), - }), - llm_request: None, - llm_response: None, - event_payload: Some(data.clone()), - event_category: mark - .category() - .map(|category| category.as_str().to_string()), - event_category_profile: mark - .category_profile() - .and_then(|profile| serde_json::to_value(profile).ok()), - tool_ancestry: Vec::new(), - tool_invocations: None, - }; - self.steps.push(AtifStep { - step_id: 0, - source: "system".to_string(), - message: mark_message(mark, data), - timestamp: Some(mark.timestamp().to_rfc3339()), - model_name: None, - reasoning_effort: None, - reasoning_content: None, - tool_calls: None, - observation: None, - metrics: None, - llm_call_count: None, - is_copied_context: None, - extra: serde_json::to_value(&extra).ok(), - }); - } - - fn handle_mark_as_tool( - &mut self, - mark: &Event, - lookups: &EventLookupMaps, - data: Option<&Json>, - ) { - self.finalize_agent_extra(); - - let ancestry = build_ancestry(mark, &lookups.name_map); - let invocation = AtifInvocationInfo { - start_timestamp: None, - end_timestamp: None, - invocation_id: Some(mark.uuid().to_string()), - status: Some("completed".to_string()), - framework: Some("nemo_relay".to_string()), - }; - let source_call_id = format!("mark:{}", mark.uuid()); - let arguments = normalize_mark_tool_arguments(data); - let mut observation_extra = event_extra(mark); - if let (Some(data), Json::Object(extra)) = (data, &mut observation_extra) { - extra.insert("event_payload".to_string(), data.clone()); - } - let extra = AtifStepExtra { - ancestry: ancestry.clone(), - invocation: Some(invocation.clone()), - llm_request: None, - llm_response: None, - event_payload: data.cloned(), - event_category: mark - .category() - .map(|category| category.as_str().to_string()), - event_category_profile: mark - .category_profile() - .and_then(|profile| serde_json::to_value(profile).ok()), - tool_ancestry: vec![ancestry], - tool_invocations: Some(vec![invocation]), - }; - - self.steps.push(AtifStep { - step_id: 0, - source: "agent".to_string(), - message: empty_message(), - timestamp: Some(mark.timestamp().to_rfc3339()), - model_name: None, - reasoning_effort: None, - reasoning_content: None, - tool_calls: Some(vec![AtifToolCall { - tool_call_id: source_call_id.clone(), - function_name: mark.name().to_string(), - // ATIF requires tool-call arguments to be a JSON object. Keep - // the original payload absence in `extra.event_payload` while - // using a schema-valid object for the projected arguments. - arguments, - extra: Some(event_extra(mark)), - }]), - observation: Some(AtifObservation { - results: vec![AtifObservationResult { - source_call_id: Some(source_call_id), - content: Some(mark_message(mark, data.unwrap_or(&Json::Null))), - subagent_trajectory_ref: None, - extra: Some(observation_extra), - }], - }), - metrics: None, - llm_call_count: Some(0), - is_copied_context: None, - extra: serde_json::to_value(&extra).ok(), - }); - } - fn handle_subagent_start(&mut self, child: &AgentScopeNode, event: &Event) { let source_call_id = self.resolve_subagent_source_call_id(event); self.flush_observations(); @@ -3300,8 +3125,6 @@ fn nearest_non_turn_agent_parent( fn events_to_trajectory( session_id: &str, agent_info: AtifAgentInfo, - mark_projection: MarkProjection, - mark_exclude_names: &[String], events: &[&Event], ) -> AtifTrajectory { let mut sorted: Vec<&Event> = events.to_vec(); @@ -3311,19 +3134,10 @@ fn events_to_trajectory( if let Some(root_uuid) = tree.choose_root(session_id) && can_use_agent_scope_tree(&tree, &sorted) { - return agent_scope_to_trajectory( - &tree, - root_uuid, - session_id, - &agent_info, - mark_projection, - mark_exclude_names, - &sorted, - true, - ); + return agent_scope_to_trajectory(&tree, root_uuid, session_id, &agent_info, &sorted, true); } - let steps = events_to_steps(&sorted, mark_projection, mark_exclude_names); + let steps = events_to_steps(&sorted); trajectory_from_parts( session_id.to_string(), Some(session_id.to_string()), @@ -3358,28 +3172,19 @@ fn is_step_event(event: &Event) -> bool { "scope", Some(crate::api::event::ScopeCategory::End), Some("tool") - ) | ("mark", _, _) + ) ) } -#[allow(clippy::too_many_arguments)] fn agent_scope_to_trajectory( tree: &AgentScopeTree, agent_uuid: Uuid, session_id: &str, agent_info: &AtifAgentInfo, - mark_projection: MarkProjection, - mark_exclude_names: &[String], sorted_events: &[&Event], is_root: bool, ) -> AtifTrajectory { - let mut steps = events_to_steps_for_agent( - sorted_events, - tree, - agent_uuid, - mark_projection, - mark_exclude_names, - ); + let mut steps = events_to_steps_for_agent(sorted_events, tree, agent_uuid); let subagent_trajectories = tree .nodes .get(&agent_uuid) @@ -3392,8 +3197,6 @@ fn agent_scope_to_trajectory( *child_uuid, session_id, agent_info, - mark_projection, - mark_exclude_names, sorted_events, false, ) @@ -3467,15 +3270,9 @@ fn events_to_steps_for_agent( events: &[&Event], tree: &AgentScopeTree, agent_uuid: Uuid, - mark_projection: MarkProjection, - mark_exclude_names: &[String], ) -> Vec { let lookups = EventLookupMaps::from_events_for_agent(events, tree, agent_uuid); - let mut state = StepConversionState { - mark_projection, - mark_exclude_names: mark_exclude_names.to_vec(), - ..StepConversionState::default() - }; + let mut state = StepConversionState::default(); for event in events { if let Some(child) = tree.direct_child_for_start(agent_uuid, event) { @@ -3509,21 +3306,12 @@ fn events_to_steps_for_agent( /// step with multiple results /// 4. Tool End observation results are correlated with the preceding LLM End's /// promoted tool_calls by function name → `source_call_id`. -/// 5. Mark events → system steps if they carry data. -/// 6. Scope Start/End → skipped. -fn events_to_steps( - events: &[&Event], - mark_projection: MarkProjection, - mark_exclude_names: &[String], -) -> Vec { +/// 5. Mark and other Scope Start/End events → skipped. +fn events_to_steps(events: &[&Event]) -> Vec { let mut sorted: Vec<&Event> = events.to_vec(); sorted.sort_by_key(|e| *e.timestamp()); let lookups = EventLookupMaps::from_events(&sorted); - let mut state = StepConversionState { - mark_projection, - mark_exclude_names: mark_exclude_names.to_vec(), - ..StepConversionState::default() - }; + let mut state = StepConversionState::default(); for event in &sorted { state.handle_event(event, &lookups); @@ -3532,43 +3320,6 @@ fn events_to_steps( state.finish() } -fn is_empty_mark_payload(data: &Json) -> bool { - data.is_null() || data.as_object().is_some_and(|object| object.is_empty()) -} - -fn normalize_mark_tool_arguments(data: Option<&Json>) -> Json { - match data { - None | Some(Json::Null) => Json::Object(serde_json::Map::new()), - Some(Json::Object(_)) => data.cloned().expect("mark data is present"), - Some(value) => { - let mut object = serde_json::Map::new(); - object.insert("value".to_string(), value.clone()); - Json::Object(object) - } - } -} - -// A runtime mark is point-in-time telemetry rather than a scoped call with start/end events. Agent -// hook adapters use marks for lifecycle notifications that do not map to first-class ATIF step -// types, for example hook-only status updates or synthetic fallback events. The ATIF step message -// stays schema-compatible while the original payload is preserved in `Step.extra.event_payload`. -fn mark_message(mark: &Event, _data: &Json) -> Json { - Json::String(mark_hook_event_name(mark).unwrap_or_default()) -} - -// Prefer the adapter-provided hook name because the runtime mark name may be a generic bucket such -// as `hook_mark` or a synthetic fallback like `subagent_end_without_start`. Falling back to the mark -// name keeps non-hook marks readable without making this exporter depend on any one agent adapter. -fn mark_hook_event_name(mark: &Event) -> Option { - mark.metadata() - .and_then(Json::as_object) - .and_then(|metadata| metadata.get("hook_event_name")) - .and_then(Json::as_str) - .filter(|name| !name.is_empty()) - .map(ToOwned::to_owned) - .or_else(|| Some(mark.name().to_string()).filter(|name| !name.is_empty())) -} - fn is_start_event(event: &Event) -> bool { event.scope_category() == Some(crate::api::event::ScopeCategory::Start) } diff --git a/crates/core/src/observability/mod.rs b/crates/core/src/observability/mod.rs index f202b10c3..f2dd3a352 100644 --- a/crates/core/src/observability/mod.rs +++ b/crates/core/src/observability/mod.rs @@ -27,7 +27,7 @@ pub mod plugin_component; /// /// Marks remain canonical ATOF events regardless of this setting. Exporters /// apply the selected projection only when translating those events into a -/// downstream trajectory or trace format. +/// downstream trace format. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "snake_case")] @@ -35,11 +35,11 @@ pub enum MarkProjection { /// Use each exporter’s native handling for marks. #[default] Inherit, - /// Force marks into ATIF system steps or OTEL span events. + /// Force marks into exporter-native trace span events. Event, - /// Render non-excluded marks as deterministic ATIF tool steps or - /// zero-duration OTEL child spans so trace-tree consumers can display them - /// directly. High-volume `llm.chunk` marks remain exporter-native events. + /// Render non-excluded marks as zero-duration trace child spans so + /// trace-tree consumers can display them directly. High-volume + /// `llm.chunk` marks remain exporter-native events. Tool, } @@ -84,16 +84,6 @@ pub(crate) fn effective_mark_projection( } } -pub(crate) fn is_llm_chunk_mark(event: &crate::api::event::Event) -> bool { - event.name() == "llm.chunk" - || event - .metadata() - .and_then(crate::json::Json::as_object) - .and_then(|metadata| metadata.get("hook_event_name")) - .and_then(crate::json::Json::as_str) - == Some("llm.chunk") -} - #[cfg(all(test, feature = "otel", feature = "openinference"))] #[path = "../../tests/unit/observability/exporter_parity_tests.rs"] mod exporter_parity_tests; diff --git a/crates/core/src/observability/plugin_component.rs b/crates/core/src/observability/plugin_component.rs index 9053d7585..8481d0640 100644 --- a/crates/core/src/observability/plugin_component.rs +++ b/crates/core/src/observability/plugin_component.rs @@ -228,13 +228,6 @@ pub struct AtifSectionConfig { /// Default model name. #[serde(default = "default_model_name")] pub model_name: String, - /// Representation used for mark events: `inherit`, `event`, or `tool`. - #[serde(default)] - #[cfg_attr(feature = "schema", schemars(schema_with = "mark_projection_schema"))] - pub mark_projection: MarkProjection, - /// Mark names excluded from tool projection. Defaults to `llm.chunk`. - #[serde(default = "default_mark_exclude_names")] - pub mark_exclude_names: Vec, /// Tool definitions available to the agent. #[serde(default, skip_serializing_if = "Option::is_none")] pub tool_definitions: Option>, @@ -271,8 +264,6 @@ impl Default for AtifSectionConfig { agent_name: default_agent_name(), agent_version: default_agent_version(), model_name: default_model_name(), - mark_projection: MarkProjection::default(), - mark_exclude_names: default_mark_exclude_names(), tool_definitions: None, extra: None, output_directory: None, @@ -495,8 +486,6 @@ crate::editor_config! { agent_name => { label: "agent_name", kind: String }, agent_version => { label: "agent_version", kind: String }, model_name => { label: "model_name", kind: String }, - mark_projection => { label: "mark_projection", kind: Enum, values: ["inherit", "event", "tool"] }, - mark_exclude_names => { label: "mark_exclude_names", kind: Json }, tool_definitions => { label: "tool_definitions", kind: Json, optional: true }, extra => { label: "extra", kind: Json, optional: true }, output_directory => { label: "output_directory", kind: String, optional: true }, @@ -959,9 +948,7 @@ impl AtifDispatcher { // subscriber is attached after that start event has already been // emitted. let session_id = event.uuid().to_string(); - let exporter = AtifExporter::new(session_id.clone(), self.agent_info()) - .with_mark_projection(self.config.mark_projection) - .with_mark_exclude_names(self.config.mark_exclude_names.clone()); + let exporter = AtifExporter::new(session_id.clone(), self.agent_info()); (exporter.subscriber())(event); let (filename, local_path) = self.prepare_destination(&session_id); self.scope_owners.insert(event.uuid(), event.uuid()); @@ -1526,8 +1513,6 @@ fn validate_observability_section_fields( "agent_name", "agent_version", "model_name", - "mark_projection", - "mark_exclude_names", "tool_definitions", "extra", "output_directory", diff --git a/crates/core/tests/integration/middleware_tests.rs b/crates/core/tests/integration/middleware_tests.rs index 0b184ec72..ea0769132 100644 --- a/crates/core/tests/integration/middleware_tests.rs +++ b/crates/core/tests/integration/middleware_tests.rs @@ -58,7 +58,7 @@ use nemo_relay::error::FlowError; #[cfg(all(feature = "otel", feature = "openinference"))] use nemo_relay::observability::MarkProjection; #[cfg(all(feature = "otel", feature = "openinference"))] -use nemo_relay::observability::atif::{AtifAgentInfo, AtifExporter, AtifStepExtra}; +use nemo_relay::observability::atif::{AtifAgentInfo, AtifExporter}; #[cfg(all(feature = "otel", feature = "openinference"))] use nemo_relay::observability::openinference::OpenInferenceSubscriber; #[cfg(all(feature = "otel", feature = "openinference"))] @@ -801,7 +801,7 @@ async fn test_tool_execution_outcome_marks_follow_end_with_tool_parentage() { #[cfg(all(feature = "otel", feature = "openinference"))] #[tokio::test] -async fn test_managed_tool_pending_marks_project_through_all_exporters() { +async fn test_managed_tool_pending_marks_project_through_trace_exporters_only() { let _lock = TEST_MUTEX.lock().unwrap(); reset_global(); setup_isolated_thread(); @@ -823,8 +823,7 @@ async fn test_managed_tool_pending_marks_project_through_all_exporters() { tool_definitions: None, extra: None, }, - ) - .with_mark_projection(MarkProjection::Tool); + ); register_subscriber("managed_tool_projection_atif", atif.subscriber()).unwrap(); let otel_exporter = InMemorySpanExporterBuilder::new().build(); @@ -910,37 +909,13 @@ async fn test_managed_tool_pending_marks_project_through_all_exporters() { drop(captured); let trajectory = atif.export().unwrap(); - let mark_step = trajectory - .steps - .iter() - .find(|step| { - step.tool_calls.as_deref().is_some_and(|calls| { - calls - .iter() - .any(|call| call.function_name == "plugin.output_compacted") - }) + assert!(trajectory.steps.iter().all(|step| { + !step.tool_calls.as_deref().is_some_and(|calls| { + calls + .iter() + .any(|call| call.function_name == "plugin.output_compacted") }) - .unwrap(); - assert_eq!(mark_step.llm_call_count, Some(0)); - let projected_call = &mark_step.tool_calls.as_ref().unwrap()[0]; - assert_eq!(projected_call.arguments, json!({"saved_tokens": 12})); - let step_extra: AtifStepExtra = - serde_json::from_value(mark_step.extra.clone().unwrap()).unwrap(); - assert_eq!(step_extra.event_category.as_deref(), Some("custom")); - assert_eq!( - step_extra.event_category_profile, - Some(json!({"subtype": "example.compaction"})) - ); - assert_eq!( - projected_call.extra.as_ref().unwrap()["metadata"], - json!({"source": "test"}) - ); - assert_eq!( - mark_step.observation.as_ref().unwrap().results[0] - .source_call_id - .as_deref(), - Some(projected_call.tool_call_id.as_str()) - ); + })); otel.force_flush().unwrap(); let otel_spans = otel_exporter.get_finished_spans().unwrap(); diff --git a/crates/core/tests/unit/atif_tests.rs b/crates/core/tests/unit/atif_tests.rs index 763c3140d..6a994b27e 100644 --- a/crates/core/tests/unit/atif_tests.rs +++ b/crates/core/tests/unit/atif_tests.rs @@ -1789,7 +1789,7 @@ fn test_exporter_openclaw_placeholder_replay_preserves_empty_user_step_and_raw_r } #[test] -fn test_exporter_openclaw_timing_marks_become_system_steps_with_payloads() { +fn test_exporter_ignores_openclaw_timing_marks() { let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); let base = base_timestamp(); @@ -1821,240 +1821,14 @@ fn test_exporter_openclaw_timing_marks_become_system_steps_with_payloads() { set_event_timestamp(&mut ambiguous, base); set_event_timestamp(&mut unpaired, base + chrono::Duration::milliseconds(1)); - { - let mut state = exporter.state.lock().unwrap(); - state.events.extend([ambiguous, unpaired]); - } - - let trajectory = exporter.export().unwrap(); - assert_atif_v17_shape(&trajectory); - assert_eq!(trajectory.steps.len(), 2); - - let ambiguous_step = &trajectory.steps[0]; - assert_eq!(ambiguous_step.source, "system"); - assert_eq!( - ambiguous_step.message, - json!("openclaw.model_call_timing_ambiguous") - ); - let ambiguous_extra: AtifStepExtra = - serde_json::from_value(ambiguous_step.extra.clone().unwrap()).unwrap(); - assert_eq!( - ambiguous_extra.event_payload.as_ref(), - Some(&ambiguous_payload) - ); - - let unpaired_step = &trajectory.steps[1]; - assert_eq!(unpaired_step.source, "system"); - assert_eq!( - unpaired_step.message, - json!("openclaw.model_call_timing_unpaired") - ); - let unpaired_extra: AtifStepExtra = - serde_json::from_value(unpaired_step.extra.clone().unwrap()).unwrap(); - assert_eq!( - unpaired_extra.event_payload.as_ref(), - Some(&unpaired_payload) - ); -} - -#[test] -fn test_exporter_tool_projection_renders_generic_mark_as_deterministic_tool_step() { - let root_uuid = Uuid::now_v7(); - let mark_uuid = Uuid::now_v7(); - let exporter = AtifExporter::new(root_uuid.to_string(), make_agent_info()) - .with_mark_projection(MarkProjection::Tool); - let base = base_timestamp(); - let payload = json!({"count": 3, "status": "compacted"}); - - let mut root_start = event_builder(root_uuid, EventType::Start) - .name("neutral-agent") - .scope_type(ScopeType::Agent) - .build(); - let mut mark = Event::Mark(MarkEvent::new( - BaseEvent::builder() - .parent_uuid(root_uuid) - .uuid(mark_uuid) - .name("plugin.output_compacted") - .data(payload.clone()) - .build(), - Some(EventCategory::custom()), - Some( - CategoryProfile::builder() - .subtype("example.compaction") - .build(), - ), - )); - let mut root_end = event_builder(root_uuid, EventType::End) - .name("neutral-agent") - .scope_type(ScopeType::Agent) - .build(); - set_event_timestamp(&mut root_start, base); - set_event_timestamp(&mut mark, base + chrono::Duration::milliseconds(1)); - set_event_timestamp(&mut root_end, base + chrono::Duration::milliseconds(2)); - - exporter - .state - .lock() - .unwrap() - .events - .extend([root_start, mark, root_end]); - - let trajectory = exporter.export().unwrap(); - assert_atif_v17_shape(&trajectory); - assert_eq!(trajectory.steps.len(), 1); - - let step = &trajectory.steps[0]; - assert_eq!(step.source, "agent"); - assert_eq!(step.llm_call_count, Some(0)); - assert_eq!( - step.timestamp.as_deref(), - Some("2026-01-01T00:00:00.001+00:00") - ); - let tool_call = &step.tool_calls.as_ref().unwrap()[0]; - assert_eq!(tool_call.function_name, "plugin.output_compacted"); - assert_eq!(tool_call.arguments, payload); - - let result = &step.observation.as_ref().unwrap().results[0]; - assert_eq!( - result.source_call_id.as_deref(), - Some(tool_call.tool_call_id.as_str()) - ); - assert_eq!(result.content, Some(json!("plugin.output_compacted"))); - - let extra: AtifStepExtra = serde_json::from_value(step.extra.clone().unwrap()).unwrap(); - assert_eq!(extra.ancestry.function_id, mark_uuid.to_string()); - assert_eq!(extra.ancestry.parent_id, Some(root_uuid.to_string())); - assert_eq!( - extra.event_payload, - Some(json!({"count": 3, "status": "compacted"})) - ); - assert_eq!(extra.event_category.as_deref(), Some("custom")); - assert_eq!( - extra.event_category_profile.as_ref().unwrap()["subtype"], - json!("example.compaction") - ); -} - -#[test] -fn test_exporter_tool_projection_exclusion_keeps_mark_as_system_step() { - let root_uuid = Uuid::now_v7(); - let exporter = AtifExporter::new(root_uuid.to_string(), make_agent_info()) - .with_mark_projection(MarkProjection::Tool) - .with_mark_exclude_names(["plugin.output_compacted"]); - let mark = Event::Mark(MarkEvent::new( - BaseEvent::builder() - .parent_uuid(root_uuid) - .name("plugin.output_compacted") - .data(json!({"count": 3})) - .build(), - Some(EventCategory::custom()), - Some( - CategoryProfile::builder() - .subtype("example.compaction") - .build(), - ), - )); - - exporter.state.lock().unwrap().events.push(mark); - - let trajectory = exporter.export().unwrap(); - assert_eq!(trajectory.steps.len(), 1); - assert_eq!(trajectory.steps[0].source, "system"); - assert!(trajectory.steps[0].tool_calls.is_none()); - assert_eq!( - trajectory.steps[0].message, - json!("plugin.output_compacted") - ); -} - -#[test] -fn test_exporter_tool_projection_keeps_data_less_and_empty_marks_visible() { - let root_uuid = Uuid::now_v7(); - let exporter = AtifExporter::new(root_uuid.to_string(), make_agent_info()) - .with_mark_projection(MarkProjection::Tool); - let base = base_timestamp(); - - let mut root_start = event_builder(root_uuid, EventType::Start) - .name("neutral-agent") - .scope_type(ScopeType::Agent) - .build(); - let mut data_less_mark = Event::Mark(MarkEvent::new( - BaseEvent::builder() - .parent_uuid(root_uuid) - .name("plugin.checkpoint") - .build(), - None, - None, - )); - let mut empty_mark = Event::Mark(MarkEvent::new( - BaseEvent::builder() - .parent_uuid(root_uuid) - .name("plugin.empty_checkpoint") - .data(json!({})) - .build(), - None, - None, - )); - let mut scalar_mark = Event::Mark(MarkEvent::new( - BaseEvent::builder() - .parent_uuid(root_uuid) - .name("plugin.scalar_checkpoint") - .data(json!("checkpoint complete")) - .build(), - None, - None, - )); - let mut root_end = event_builder(root_uuid, EventType::End) - .name("neutral-agent") - .scope_type(ScopeType::Agent) - .build(); - set_event_timestamp(&mut root_start, base); - set_event_timestamp( - &mut data_less_mark, - base + chrono::Duration::milliseconds(1), - ); - set_event_timestamp(&mut empty_mark, base + chrono::Duration::milliseconds(2)); - set_event_timestamp(&mut scalar_mark, base + chrono::Duration::milliseconds(3)); - set_event_timestamp(&mut root_end, base + chrono::Duration::milliseconds(4)); - - exporter.state.lock().unwrap().events.extend([ - root_start, - data_less_mark, - empty_mark, - scalar_mark, - root_end, - ]); + let subscriber = exporter.subscriber(); + subscriber(&ambiguous); + subscriber(&unpaired); + assert!(exporter.state.lock().unwrap().events.is_empty()); let trajectory = exporter.export().unwrap(); assert_atif_v17_shape(&trajectory); - assert_eq!(trajectory.steps.len(), 3); - - let data_less_tool = &trajectory.steps[0].tool_calls.as_ref().unwrap()[0]; - assert_eq!(data_less_tool.function_name, "plugin.checkpoint"); - assert_eq!(data_less_tool.arguments, json!({})); - let data_less_extra: AtifStepExtra = - serde_json::from_value(trajectory.steps[0].extra.clone().unwrap()).unwrap(); - assert_eq!(data_less_extra.event_payload, None); - - let empty_tool = &trajectory.steps[1].tool_calls.as_ref().unwrap()[0]; - assert_eq!(empty_tool.function_name, "plugin.empty_checkpoint"); - assert_eq!(empty_tool.arguments, json!({})); - let empty_extra: AtifStepExtra = - serde_json::from_value(trajectory.steps[1].extra.clone().unwrap()).unwrap(); - assert_eq!(empty_extra.event_payload, Some(json!({}))); - - let scalar_tool = &trajectory.steps[2].tool_calls.as_ref().unwrap()[0]; - assert_eq!(scalar_tool.function_name, "plugin.scalar_checkpoint"); - assert_eq!( - scalar_tool.arguments, - json!({"value": "checkpoint complete"}) - ); - let scalar_extra: AtifStepExtra = - serde_json::from_value(trajectory.steps[2].extra.clone().unwrap()).unwrap(); - assert_eq!( - scalar_extra.event_payload, - Some(json!("checkpoint complete")) - ); + assert!(trajectory.steps.is_empty()); } #[test] @@ -2819,16 +2593,15 @@ fn test_exporter_tool_call_id_linking() { } #[test] -fn test_exporter_mark_steps_include_hook_name_and_ancestry() { +fn test_exporter_ignores_marks_with_hook_metadata() { let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); let agent_uuid = Uuid::now_v7(); - let mark_uuid = Uuid::now_v7(); let agent_start = event_builder(agent_uuid, EventType::Start) .name("hermes") .scope_type(ScopeType::Agent) .build(); - let mark = event_builder(mark_uuid, EventType::Mark) + let mark = event_builder(Uuid::now_v7(), EventType::Mark) .name("subagent_end_without_start") .parent_uuid(agent_uuid) .data(json!({ @@ -2849,25 +2622,7 @@ fn test_exporter_mark_steps_include_hook_name_and_ancestry() { } let trajectory = exporter.export().unwrap(); - assert_eq!(trajectory.steps.len(), 1); - - let step = &trajectory.steps[0]; - assert_eq!(step.source, "system"); - assert_eq!(step.message, json!("subagent_stop")); - - let extra: AtifStepExtra = serde_json::from_value(step.extra.clone().unwrap()).unwrap(); - assert_eq!( - extra.event_payload.as_ref().unwrap()["extra"]["subagent_id"], - json!("worker-1") - ); - assert_eq!(extra.ancestry.function_id, mark_uuid.to_string()); - assert_eq!(extra.ancestry.function_name, "subagent_end_without_start"); - assert_eq!(extra.ancestry.parent_id, Some(agent_uuid.to_string())); - assert_eq!(extra.ancestry.parent_name, Some("hermes".to_string())); - assert_eq!( - extra.invocation.as_ref().unwrap().invocation_id, - Some(mark_uuid.to_string()) - ); + assert!(trajectory.steps.is_empty()); } #[test] @@ -3033,10 +2788,11 @@ fn test_exporter_attaches_subagent_ref_to_delegating_tool_observation() { "tool_call_id": "call_delegate" })) .build(); - let mut child_mark = event_builder(child_mark_uuid, EventType::Mark) - .name("worker-started") + let mut child_mark = event_builder(child_mark_uuid, EventType::End) + .name("worker-llm") + .scope_type(ScopeType::Llm) .parent_uuid(child_uuid) - .data(json!({"status": "started"})) + .output(json!({"content": "worker started"})) .build(); let mut child_end = event_builder(child_uuid, EventType::End) .name("worker-agent") @@ -3134,10 +2890,11 @@ fn test_exporter_synthesizes_tool_call_for_active_subagent_dispatch() { .parent_uuid(root_uuid) .metadata(json!({"session_id": "child-session"})) .build(); - let mut child_mark = event_builder(child_mark_uuid, EventType::Mark) - .name("worker-started") + let mut child_mark = event_builder(child_mark_uuid, EventType::End) + .name("worker-llm") + .scope_type(ScopeType::Llm) .parent_uuid(child_uuid) - .data(json!({"status": "started"})) + .output(json!({"content": "worker started"})) .build(); let mut child_end = event_builder(child_uuid, EventType::End) .name("worker-agent") @@ -3457,11 +3214,9 @@ fn test_exporter_renumbers_after_pruning_empty_subagent_ref_step() { let trajectory = exporter.export().unwrap(); assert_atif_v17_shape(&trajectory); - assert_eq!(trajectory.steps.len(), 2); + assert_eq!(trajectory.steps.len(), 1); assert_eq!(trajectory.steps[0].step_id, 1); assert_eq!(trajectory.steps[0].source, "agent"); - assert_eq!(trajectory.steps[1].step_id, 2); - assert_eq!(trajectory.steps[1].source, "system"); assert!(trajectory.subagent_trajectories.is_none()); let serialized = serde_json::to_value(&trajectory).unwrap(); assert!(!serialized.to_string().contains("subagent_trajectory_ref")); @@ -3491,10 +3246,11 @@ fn test_exporter_embeds_recursive_subagent_trajectories() { .parent_uuid(child_uuid) .metadata(json!({"session_id": "grandchild-session"})) .build(); - let mut grandchild_mark = event_builder(Uuid::now_v7(), EventType::Mark) - .name("deep-note") + let mut grandchild_mark = event_builder(Uuid::now_v7(), EventType::End) + .name("deep-llm") + .scope_type(ScopeType::Llm) .parent_uuid(grandchild_uuid) - .data(json!({"status": "ok"})) + .output(json!({"content": "deep-note"})) .build(); let mut grandchild_end = event_builder(grandchild_uuid, EventType::End) .name("deep-worker") @@ -3557,13 +3313,10 @@ fn test_exporter_embeds_recursive_subagent_trajectories() { assert_eq!(grandchild.steps.len(), 1); assert_eq!(grandchild.steps[0].step_id, 1); assert_eq!(grandchild.steps[0].message, json!("deep-note")); - let extra: AtifStepExtra = - serde_json::from_value(grandchild.steps[0].extra.clone().unwrap()).unwrap(); - assert_eq!(extra.event_payload.as_ref().unwrap()["status"], json!("ok")); } #[test] -fn test_exporter_skips_empty_mark_payloads() { +fn test_exporter_skips_all_mark_payloads() { let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); { @@ -3574,6 +3327,12 @@ fn test_exporter_skips_empty_mark_payloads() { .data(json!({})) .build(), ); + state.events.push( + event_builder(Uuid::now_v7(), EventType::Mark) + .name("populated") + .data(json!({"status": "ok"})) + .build(), + ); state.events.push( event_builder(Uuid::now_v7(), EventType::Mark) .name("empty-null") @@ -3591,7 +3350,7 @@ fn test_exporter_skips_empty_mark_payloads() { } #[test] -fn test_exporter_skips_llm_chunk_marks() { +fn test_exporter_skips_marks_regardless_of_name() { let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); { @@ -3619,26 +3378,7 @@ fn test_exporter_skips_llm_chunk_marks() { let trajectory = exporter.export().unwrap(); - assert_eq!(trajectory.steps.len(), 1); - assert_eq!(trajectory.steps[0].message, json!("agent.status")); -} - -#[test] -fn test_exporter_event_projection_includes_llm_chunk_marks() { - let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()) - .with_mark_projection(MarkProjection::Event); - - exporter.state.lock().unwrap().events.push( - event_builder(Uuid::now_v7(), EventType::Mark) - .name("llm.chunk") - .data(json!({"delta": "partial"})) - .build(), - ); - - let trajectory = exporter.export().unwrap(); - assert_eq!(trajectory.steps.len(), 1); - assert_eq!(trajectory.steps[0].source, "system"); - assert_eq!(trajectory.steps[0].message, json!("llm.chunk")); + assert!(trajectory.steps.is_empty()); } #[test] @@ -4219,8 +3959,9 @@ fn test_exporter_clear() { { let mut state = exporter.state.lock().unwrap(); state.events.push( - event_builder(Uuid::now_v7(), EventType::Mark) - .data(json!("test")) + event_builder(Uuid::now_v7(), EventType::End) + .scope_type(ScopeType::Llm) + .output(json!("test")) .build(), ); } diff --git a/crates/core/tests/unit/observability/openinference_tests.rs b/crates/core/tests/unit/observability/openinference_tests.rs index 77fdb17dc..f9581cf6b 100644 --- a/crates/core/tests/unit/observability/openinference_tests.rs +++ b/crates/core/tests/unit/observability/openinference_tests.rs @@ -2313,7 +2313,7 @@ fn late_parented_marks_reuse_completed_parent_trace_context() { )); processor.process(&make_mark_event( Some(tool_uuid), - "visor.tool_output_compressed", + "plugin.output_compacted", Some(json!({"estimated_tokens_saved": 42})), )); processor.force_flush().unwrap(); @@ -2326,7 +2326,7 @@ fn late_parented_marks_reuse_completed_parent_trace_context() { .unwrap(); let mark_span = spans .iter() - .find(|span| span.name.as_ref() == "mark:visor.tool_output_compressed") + .find(|span| span.name.as_ref() == "mark:plugin.output_compacted") .unwrap(); assert_eq!( diff --git a/crates/core/tests/unit/observability/otel_tests.rs b/crates/core/tests/unit/observability/otel_tests.rs index ff164d527..b03e3e7f5 100644 --- a/crates/core/tests/unit/observability/otel_tests.rs +++ b/crates/core/tests/unit/observability/otel_tests.rs @@ -987,7 +987,7 @@ fn late_parented_marks_reuse_completed_parent_trace_context() { )); processor.process(&make_mark_event( Some(tool_uuid), - "visor.tool_output_compressed", + "plugin.output_compacted", Some(json!({"estimated_tokens_saved": 42})), )); processor.force_flush().unwrap(); @@ -1000,7 +1000,7 @@ fn late_parented_marks_reuse_completed_parent_trace_context() { .unwrap(); let mark_span = spans .iter() - .find(|span| span.name.as_ref() == "mark:visor.tool_output_compressed") + .find(|span| span.name.as_ref() == "mark:plugin.output_compacted") .unwrap(); assert_eq!( diff --git a/crates/core/tests/unit/observability/plugin_component_tests.rs b/crates/core/tests/unit/observability/plugin_component_tests.rs index 8d9b2652a..6a78c3369 100644 --- a/crates/core/tests/unit/observability/plugin_component_tests.rs +++ b/crates/core/tests/unit/observability/plugin_component_tests.rs @@ -216,8 +216,6 @@ fn default_config_and_component_conversion_cover_public_shape() { assert_eq!(atif.agent_name, "NeMo Relay"); assert_eq!(atif.agent_version, env!("CARGO_PKG_VERSION")); assert_eq!(atif.model_name, "unknown"); - assert_eq!(atif.mark_projection, MarkProjection::Inherit); - assert_eq!(atif.mark_exclude_names, vec!["llm.chunk"]); assert_eq!(atif.filename_template, "nemo-relay-atif-{session_id}.json"); let otlp = OtlpSectionConfig::default(); @@ -243,24 +241,13 @@ fn default_config_and_component_conversion_cover_public_shape() { } #[test] -fn mark_projection_parses_per_exporter_and_rejects_unknown_values() { - let atif: AtifSectionConfig = serde_json::from_value(json!({ - "mark_projection": "tool" - })) - .unwrap(); +fn mark_projection_parses_for_otlp_and_rejects_unknown_values() { let otlp: OtlpSectionConfig = serde_json::from_value(json!({ "mark_projection": "tool" })) .unwrap(); - assert_eq!(atif.mark_projection, MarkProjection::Tool); assert_eq!(otlp.mark_projection, MarkProjection::Tool); - let inherited: AtifSectionConfig = serde_json::from_value(json!({ - "mark_projection": "inherit" - })) - .unwrap(); - assert_eq!(inherited.mark_projection, MarkProjection::Inherit); - let custom_exclusions: OtlpSectionConfig = serde_json::from_value(json!({ "mark_projection": "tool", "mark_exclude_names": ["notification", "hook_mark"] @@ -284,6 +271,14 @@ fn mark_projection_parses_per_exporter_and_rejects_unknown_values() { assert!(report.diagnostics.iter().any(|diagnostic| diagnostic.code == "observability.invalid_plugin_config" && diagnostic.message.contains("unknown variant `span`"))); + + let report = validate_plugin_config(&plugin_config(json!({ + "atif": {"mark_projection": "tool"} + }))); + assert!(report.diagnostics.iter().any(|diagnostic| { + diagnostic.code == "observability.unknown_field" + && diagnostic.field.as_deref() == Some("mark_projection") + })); } #[cfg(feature = "schema")] @@ -976,7 +971,7 @@ fn atif_routes_global_descendant_events_by_parent_uuid() { } #[test] -fn atif_keeps_openclaw_child_only_fallback_as_a_top_level_trajectory() { +fn atif_writes_openclaw_child_only_fallback_without_mark_steps() { let _guard = crate::observability::test_mutex().lock().unwrap(); reset_runtime(); let dir = temp_dir("observability-atif-openclaw-child-fallback"); @@ -1080,8 +1075,7 @@ fn atif_keeps_openclaw_child_only_fallback_as_a_top_level_trajectory() { let value: Json = serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap(); assert_eq!(value["trajectory_id"], child_uuid.to_string()); - assert_eq!(value["steps"].as_array().unwrap().len(), 1); - assert_eq!(value["steps"][0]["message"], "worker-started"); + assert!(value["steps"].as_array().unwrap().is_empty()); assert!( value.get("subagent_trajectories").is_none() || value["subagent_trajectories"].is_null() ); diff --git a/crates/node/observability.d.ts b/crates/node/observability.d.ts index 53d3319cc..eb15be134 100644 --- a/crates/node/observability.d.ts +++ b/crates/node/observability.d.ts @@ -47,8 +47,6 @@ export interface AtifConfig { agent_name?: string; agent_version?: string; model_name?: string; - mark_projection?: 'inherit' | 'event' | 'tool'; - mark_exclude_names?: string[]; tool_definitions?: Record[]; extra?: Record; output_directory?: string; diff --git a/crates/node/observability.js b/crates/node/observability.js index 05f7e72b6..97e5c8415 100644 --- a/crates/node/observability.js +++ b/crates/node/observability.js @@ -43,8 +43,6 @@ function atifConfig(config = {}) { enabled: false, agent_name: 'NeMo Relay', model_name: 'unknown', - mark_projection: 'inherit', - mark_exclude_names: ['llm.chunk'], filename_template: 'nemo-relay-atif-{session_id}.json', ...config, }; diff --git a/crates/node/tests/observability_plugin_tests.mjs b/crates/node/tests/observability_plugin_tests.mjs index c384ebe33..94b825031 100644 --- a/crates/node/tests/observability_plugin_tests.mjs +++ b/crates/node/tests/observability_plugin_tests.mjs @@ -25,8 +25,6 @@ describe('observability plugin helpers', () => { enabled: false, agent_name: 'NeMo Relay', model_name: 'unknown', - mark_projection: 'inherit', - mark_exclude_names: ['llm.chunk'], filename_template: 'nemo-relay-atif-{session_id}.json', }); assert.deepEqual(observability.otlpConfig(), { @@ -39,7 +37,6 @@ describe('observability plugin helpers', () => { service_name: 'nemo-relay', timeout_millis: 3000, }); - assert.equal(observability.atifConfig({ mark_projection: 'tool' }).mark_projection, 'tool'); assert.equal(observability.otlpConfig({ mark_projection: 'tool' }).mark_projection, 'tool'); const component = observability.ComponentSpec({ version: 1, atof: observability.atofConfig() }); diff --git a/docs/configure-plugins/observability/atif.mdx b/docs/configure-plugins/observability/atif.mdx index bd8d3abdd..d56617c69 100644 --- a/docs/configure-plugins/observability/atif.mdx +++ b/docs/configure-plugins/observability/atif.mdx @@ -43,6 +43,11 @@ This configuration writes a trajectory file such as `logs/trajectory-.json` for each top-level Agent scope or supported coding-agent turn scope. +ATIF contains agent interaction steps, tool calls, and observations. Relay mark +events are point-in-time telemetry rather than trajectory steps, so the ATIF +exporter omits them. Use ATOF to retain the canonical mark event stream, or an +OTLP exporter when marks must remain correlated with trace data. + ## Fields The following table describes the top-level ATIF settings: @@ -53,8 +58,6 @@ The following table describes the top-level ATIF settings: | `agent_name` | `NeMo Relay` | Agent metadata written into the trajectory. | | `agent_version` | NeMo Relay crate version | Agent version metadata. | | `model_name` | `unknown` | Default model metadata when no call-level model is present. | -| `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces system steps; `tool` emits deterministic agent/tool steps for trajectory viewers that render tool calls but not system events. | -| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain exporter-native handling. Metadata `hook_event_name` aliases are also matched. | | `tool_definitions` | Omitted | Optional ATIF tool metadata. | | `extra` | Omitted | Optional ATIF agent metadata. | | `output_directory` | Current working directory | Directory containing trajectory files. Ignored when `storage` is non-empty. | @@ -196,21 +199,6 @@ per-destination sink failures to the failed sink. ## Expected Output -By default, marks use exporter-native handling. In ATIF, marks with payloads -remain point-in-time `system` steps with their payload, category, category -profile, timestamp, and ancestry preserved in `extra`. Set -`mark_projection = "tool"` when the target viewer requires a visible tool-call -projection. The projected step uses `llm_call_count = 0`, retains the original -mark data in `extra`, and does not change the canonical ATOF event. - -Add other event names to `mark_exclude_names` when a trajectory should keep -those marks in their native system-step representation rather than projecting -them as tool steps. The exclusion list affects only tool projection; it does -not remove mark payload or metadata. ATIF suppresses the built-in high-volume -`llm.chunk` stream receipts in `inherit` mode and when they are excluded from -`tool` projection. `event` explicitly includes payload-bearing `llm.chunk` -marks as system steps. - The exporter translates NeMo Relay lifecycle events into ATIF v1.7 trajectory data. LLM start and end events become model steps, tool events become tool calls and observations, and scope nesting contributes lineage metadata. diff --git a/docs/resources/glossary.mdx b/docs/resources/glossary.mdx index 72aa80ed3..c91106710 100644 --- a/docs/resources/glossary.mdx +++ b/docs/resources/glossary.mdx @@ -45,7 +45,7 @@ An **adaptive state backend** stores observations and learned state for adaptive **Agent Trajectory Observability Format (ATOF)** -**Agent Trajectory Observability Format (ATOF)** is the canonical event format NeMo Relay emits for scope lifecycle events and mark events. Subscribers and exporters consume ATOF events before translating them into downstream observability formats such as ATIF trajectories, OpenTelemetry traces, or OpenInference spans. +**Agent Trajectory Observability Format (ATOF)** is the canonical event format NeMo Relay emits for scope lifecycle events and mark events. Subscribers and exporters consume ATOF events before translating compatible scope data into ATIF trajectories and trace data into OpenTelemetry or OpenInference. **Annotated Request And Response Data** diff --git a/go/nemo_relay/observability_plugin.go b/go/nemo_relay/observability_plugin.go index c95af33c4..2c535a059 100644 --- a/go/nemo_relay/observability_plugin.go +++ b/go/nemo_relay/observability_plugin.go @@ -54,8 +54,6 @@ type ObservabilityAtifConfig struct { AgentName string `json:"agent_name,omitempty"` AgentVersion string `json:"agent_version,omitempty"` ModelName string `json:"model_name,omitempty"` - MarkProjection ObservabilityMarkProjection `json:"mark_projection,omitempty"` - MarkExcludeNames []string `json:"mark_exclude_names,omitempty"` ToolDefinitions []map[string]any `json:"tool_definitions,omitempty"` Extra map[string]any `json:"extra,omitempty"` OutputDirectory string `json:"output_directory,omitempty"` @@ -179,8 +177,6 @@ func NewObservabilityAtifConfig() ObservabilityAtifConfig { return ObservabilityAtifConfig{ AgentName: "NeMo Relay", ModelName: "unknown", - MarkProjection: ObservabilityMarkProjectionInherit, - MarkExcludeNames: []string{"llm.chunk"}, FilenameTemplate: "nemo-relay-atif-{session_id}.json", } } diff --git a/go/nemo_relay/observability_plugin_test.go b/go/nemo_relay/observability_plugin_test.go index 399d899f4..a69c2c0d2 100644 --- a/go/nemo_relay/observability_plugin_test.go +++ b/go/nemo_relay/observability_plugin_test.go @@ -43,7 +43,7 @@ func TestObservabilityConfigHelpers(t *testing.T) { FieldNamePolicy: "replace_dots", }} atif := NewObservabilityAtifConfig() - if atif.Enabled || atif.AgentName != "NeMo Relay" || atif.ModelName != "unknown" || atif.MarkProjection != ObservabilityMarkProjectionInherit || len(atif.MarkExcludeNames) != 1 || atif.MarkExcludeNames[0] != "llm.chunk" || atif.FilenameTemplate != "nemo-relay-atif-{session_id}.json" { + if atif.Enabled || atif.AgentName != "NeMo Relay" || atif.ModelName != "unknown" || atif.FilenameTemplate != "nemo-relay-atif-{session_id}.json" { t.Fatalf("unexpected ATIF defaults: %#v", atif) } allowHTTP := false @@ -67,7 +67,6 @@ func TestObservabilityConfigHelpers(t *testing.T) { if otlp.Enabled || otlp.MarkProjection != ObservabilityMarkProjectionInherit || len(otlp.MarkExcludeNames) != 1 || otlp.MarkExcludeNames[0] != "llm.chunk" || otlp.Transport != "http_binary" || otlp.ServiceName != "nemo-relay" || otlp.TimeoutMillis != 3000 { t.Fatalf("unexpected OTLP defaults: %#v", otlp) } - atif.MarkProjection = ObservabilityMarkProjectionTool otlp.MarkProjection = ObservabilityMarkProjectionTool config.Atof = &atof @@ -97,7 +96,7 @@ func TestObservabilityConfigHelpers(t *testing.T) { t.Fatalf("expected field_name_policy in serialized component, got %s", serialized) } assertWrappedAtifStorageConfig(t, wrapped.Config["atif"].(map[string]any)) - if wrapped.Config["atif"].(map[string]any)["mark_projection"] != "tool" || wrapped.Config["opentelemetry"].(map[string]any)["mark_projection"] != "tool" { + if wrapped.Config["opentelemetry"].(map[string]any)["mark_projection"] != "tool" { t.Fatalf("expected tool mark projection in serialized config: %#v", wrapped.Config) } } diff --git a/python/nemo_relay/observability.py b/python/nemo_relay/observability.py index 541b74be1..ad837a0a1 100644 --- a/python/nemo_relay/observability.py +++ b/python/nemo_relay/observability.py @@ -167,8 +167,6 @@ class AtifConfig: agent_name: str = "NeMo Relay" agent_version: str | None = None model_name: str = "unknown" - mark_projection: MarkProjection = "inherit" - mark_exclude_names: list[str] = field(default_factory=lambda: ["llm.chunk"]) tool_definitions: list[JsonObject] | None = None extra: JsonObject | None = None output_directory: str | None = None @@ -182,8 +180,6 @@ def to_dict(self) -> JsonObject: "agent_name": self.agent_name, "agent_version": self.agent_version, "model_name": self.model_name, - "mark_projection": self.mark_projection, - "mark_exclude_names": self.mark_exclude_names, "tool_definitions": self.tool_definitions, "extra": self.extra, "output_directory": self.output_directory, diff --git a/python/nemo_relay/observability.pyi b/python/nemo_relay/observability.pyi index a12b99852..f562a10a1 100644 --- a/python/nemo_relay/observability.pyi +++ b/python/nemo_relay/observability.pyi @@ -63,8 +63,6 @@ class AtifConfig: agent_name: str = ... agent_version: str | None = ... model_name: str = ... - mark_projection: MarkProjection = ... - mark_exclude_names: list[str] = ... tool_definitions: list[JsonObject] | None = ... extra: JsonObject | None = ... output_directory: str | None = ... diff --git a/python/tests/test_observability_plugin.py b/python/tests/test_observability_plugin.py index dcd1774be..e536fcaf8 100644 --- a/python/tests/test_observability_plugin.py +++ b/python/tests/test_observability_plugin.py @@ -34,8 +34,6 @@ def test_defaults_and_component_wrapper(self): "enabled": False, "agent_name": "NeMo Relay", "model_name": "unknown", - "mark_projection": "inherit", - "mark_exclude_names": ["llm.chunk"], "filename_template": "nemo-relay-atif-{session_id}.json", } assert OtlpConfig().to_dict() == { @@ -48,7 +46,6 @@ def test_defaults_and_component_wrapper(self): "service_name": "nemo-relay", "timeout_millis": 3000, } - assert AtifConfig(mark_projection="tool").to_dict()["mark_projection"] == "tool" assert OtlpConfig(mark_projection="tool").to_dict()["mark_projection"] == "tool" wrapped = ComponentSpec(ObservabilityConfig(atof=AtofConfig())).to_dict() From ae7737d8fd0a2ccf80f16738fb0660417cd47256 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 8 Jul 2026 10:46:57 -0600 Subject: [PATCH 11/14] fix(observability): preserve ATIF API compatibility Signed-off-by: Bryan Bednarski --- crates/cli/tests/coverage/session_tests.rs | 22 ++-------------------- crates/core/src/observability/atif.rs | 10 +++++++++- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/crates/cli/tests/coverage/session_tests.rs b/crates/cli/tests/coverage/session_tests.rs index 0a283b0bb..aa92a95e2 100644 --- a/crates/cli/tests/coverage/session_tests.rs +++ b/crates/cli/tests/coverage/session_tests.rs @@ -2896,7 +2896,7 @@ async fn hermes_post_tool_call_writes_atif_observation_with_source_call_id() { } #[tokio::test] -async fn hermes_orphan_subagent_stop_exports_readable_mark_with_lineage() { +async fn hermes_orphan_subagent_stop_does_not_create_atif_mark_step() { let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; let temp = tempfile::tempdir().unwrap(); let atif_dir = temp.path().join("atif"); @@ -2911,25 +2911,7 @@ async fn hermes_orphan_subagent_stop_exports_readable_mark_with_lineage() { let atif = read_atif_for_session(&atif_dir, "hermes-orphan"); assert!(atif["subagent_trajectories"].is_null()); let root_steps = atif["steps"].as_array().unwrap(); - assert_eq!(root_steps.len(), 1); - assert_eq!(root_steps[0]["source"], json!("system")); - assert_eq!(root_steps[0]["message"], json!("subagent_stop")); - assert_eq!( - root_steps[0]["extra"]["event_payload"]["hook_event_name"], - json!("subagent_stop") - ); - assert_eq!( - root_steps[0]["extra"]["event_payload"]["extra"]["subagent_id"], - json!("worker-1") - ); - assert_eq!( - root_steps[0]["extra"]["ancestry"]["function_name"], - json!("subagent_end_without_start") - ); - assert_eq!( - root_steps[0]["extra"]["ancestry"]["parent_name"], - json!("hermes-turn") - ); + assert!(root_steps.is_empty()); } #[tokio::test] diff --git a/crates/core/src/observability/atif.rs b/crates/core/src/observability/atif.rs index 6e1967260..96ffe8665 100644 --- a/crates/core/src/observability/atif.rs +++ b/crates/core/src/observability/atif.rs @@ -273,6 +273,12 @@ pub struct AtifStepExtra { /// Full raw LLM response payload for response-level fidelity. #[serde(skip_serializing_if = "Option::is_none")] pub llm_response: Option, + /// Legacy event payload field retained for source compatibility. + /// + /// The ATIF exporter does not translate point-in-time mark events into + /// trajectory steps, so exporter-produced steps leave this field unset. + #[serde(skip_serializing_if = "Option::is_none")] + pub event_payload: Option, /// Per-tool callable lineage, aligned with `tool_calls`. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub tool_ancestry: Vec, @@ -361,7 +367,7 @@ impl AtifExporter { pub fn subscriber(&self) -> EventSubscriberFn { let state = self.state.clone(); Arc::new(move |event: &Event| { - if event.kind() == "mark" { + if matches!(event, Event::Mark(_)) { return; } if let Ok(mut s) = state.lock() { @@ -1887,6 +1893,7 @@ impl PendingAgentStep { invocation: self.invocation.take(), llm_request: None, llm_response: self.llm_response.take(), + event_payload: None, tool_ancestry: std::mem::take(&mut self.tool_ancestry), tool_invocations: if self.tool_invocations.is_empty() { None @@ -2241,6 +2248,7 @@ impl StepConversionState { invocation: None, llm_request: Some(content.clone()), llm_response: None, + event_payload: None, tool_ancestry: Vec::new(), tool_invocations: None, }; From 948e13020ffc81130cd38187dc5e0a29675d2db1 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 8 Jul 2026 11:35:41 -0600 Subject: [PATCH 12/14] fix(observability): preserve mark exclusion semantics Signed-off-by: Bryan Bednarski --- .../observability/plugin_component_tests.rs | 8 +++++++ .../observability/openinference.mdx | 2 +- .../observability/opentelemetry.mdx | 2 +- go/nemo_relay/observability_plugin.go | 24 +++++++++++++++++++ go/nemo_relay/observability_plugin_test.go | 20 ++++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-) diff --git a/crates/core/tests/unit/observability/plugin_component_tests.rs b/crates/core/tests/unit/observability/plugin_component_tests.rs index 6a78c3369..c43d31623 100644 --- a/crates/core/tests/unit/observability/plugin_component_tests.rs +++ b/crates/core/tests/unit/observability/plugin_component_tests.rs @@ -248,6 +248,14 @@ fn mark_projection_parses_for_otlp_and_rejects_unknown_values() { .unwrap(); assert_eq!(otlp.mark_projection, MarkProjection::Tool); + let event: OtlpSectionConfig = serde_json::from_value(json!({ + "mark_projection": "event", + "mark_exclude_names": [] + })) + .unwrap(); + assert_eq!(event.mark_projection, MarkProjection::Event); + assert!(event.mark_exclude_names.is_empty()); + let custom_exclusions: OtlpSectionConfig = serde_json::from_value(json!({ "mark_projection": "tool", "mark_exclude_names": ["notification", "hook_mark"] diff --git a/docs/configure-plugins/observability/openinference.mdx b/docs/configure-plugins/observability/openinference.mdx index 4eb051ce3..accf78bad 100644 --- a/docs/configure-plugins/observability/openinference.mdx +++ b/docs/configure-plugins/observability/openinference.mdx @@ -95,7 +95,7 @@ payload, category, profile, timestamp, and parent identifiers retained as attributes. High-volume `llm.chunk` marks retain exporter-native handling by default and when excluded from tool projection. -Add other event names to `mark_exclude_names` when a backend should keep those +Add other event names to `mark_exclude_names` for a backend to keep those marks in exporter-native form rather than visible tool children. The exclusion list affects only tool projection; it does not remove mark payload or metadata. diff --git a/docs/configure-plugins/observability/opentelemetry.mdx b/docs/configure-plugins/observability/opentelemetry.mdx index 6f6e22c26..335a3789b 100644 --- a/docs/configure-plugins/observability/opentelemetry.mdx +++ b/docs/configure-plugins/observability/opentelemetry.mdx @@ -83,7 +83,7 @@ marks use `SpanKind::Internal`, carry category/profile, and payload attributes. High-volume `llm.chunk` marks retain exporter-native handling by default and when excluded from tool projection. -Add other event names to `mark_exclude_names` when a backend should keep those +Add other event names to `mark_exclude_names` for a backend to keep those marks in exporter-native form rather than visible tool children. The exclusion list affects only tool projection; it does not remove mark payload or metadata. diff --git a/go/nemo_relay/observability_plugin.go b/go/nemo_relay/observability_plugin.go index 2c535a059..c79488919 100644 --- a/go/nemo_relay/observability_plugin.go +++ b/go/nemo_relay/observability_plugin.go @@ -154,6 +154,30 @@ type ObservabilityOtlpConfig struct { TimeoutMillis uint64 `json:"timeout_millis,omitempty"` } +// MarshalJSON preserves the distinction between a nil exclusion list, which +// inherits the core default, and an explicitly empty list, which disables all +// default exclusions. +func (config ObservabilityOtlpConfig) MarshalJSON() ([]byte, error) { + type alias ObservabilityOtlpConfig + payload, err := json.Marshal(alias(config)) + if err != nil { + return nil, err + } + + var object map[string]json.RawMessage + if err := json.Unmarshal(payload, &object); err != nil { + return nil, err + } + if config.MarkExcludeNames != nil { + exclusions, err := json.Marshal(config.MarkExcludeNames) + if err != nil { + return nil, err + } + object["mark_exclude_names"] = exclusions + } + return json.Marshal(object) +} + // ObservabilityComponentSpec wraps one observability config as a top-level plugin component. type ObservabilityComponentSpec struct { Enabled bool `json:"enabled,omitempty"` diff --git a/go/nemo_relay/observability_plugin_test.go b/go/nemo_relay/observability_plugin_test.go index a69c2c0d2..7c94c33b9 100644 --- a/go/nemo_relay/observability_plugin_test.go +++ b/go/nemo_relay/observability_plugin_test.go @@ -101,6 +101,26 @@ func TestObservabilityConfigHelpers(t *testing.T) { } } +func TestObservabilityOtlpConfigPreservesExplicitEmptyMarkExclusions(t *testing.T) { + inherited, err := json.Marshal(ObservabilityOtlpConfig{}) + if err != nil { + t.Fatalf("marshal zero-value OTLP config failed: %v", err) + } + if strings.Contains(string(inherited), `"mark_exclude_names"`) { + t.Fatalf("nil exclusions should inherit the core default: %s", inherited) + } + + config := NewObservabilityOtlpConfig() + config.MarkExcludeNames = []string{} + explicitEmpty, err := json.Marshal(config) + if err != nil { + t.Fatalf("marshal OTLP config with empty exclusions failed: %v", err) + } + if !strings.Contains(string(explicitEmpty), `"mark_exclude_names":[]`) { + t.Fatalf("explicit empty exclusions should be preserved: %s", explicitEmpty) + } +} + func assertS3StorageConfig(t *testing.T, storage ObservabilityS3StorageConfig) { t.Helper() if storage.Bucket != "archive" || From a6998242457ef8915e15b3dc2850736cc53a9b23 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 8 Jul 2026 11:52:53 -0600 Subject: [PATCH 13/14] docs(observability): clarify mark projection modes Signed-off-by: Bryan Bednarski --- docs/configure-plugins/observability/openinference.mdx | 7 ++++--- docs/configure-plugins/observability/opentelemetry.mdx | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/configure-plugins/observability/openinference.mdx b/docs/configure-plugins/observability/openinference.mdx index accf78bad..423d5435f 100644 --- a/docs/configure-plugins/observability/openinference.mdx +++ b/docs/configure-plugins/observability/openinference.mdx @@ -61,7 +61,7 @@ OpenInference uses the same OTLP section shape as | Field | Default | Notes | |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | -| `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces span events; `tool` emits zero-duration OpenInference `TOOL` child spans. | +| `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces span events; `tool` emits zero-duration OpenInference `TOOL` spans, parented as children when context is available. | | `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain exporter-native handling. Metadata `hook_event_name` aliases are also matched. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | @@ -88,8 +88,8 @@ transport metadata. The default `inherit` projection preserves exporter-native mark handling: a mark with an active parent span is a span event, while an orphan mark is a standalone zero-duration mark span. `mark_projection = "event"` explicitly -selects that representation. With `mark_projection = "tool"`, each -non-streaming mark becomes a zero-duration span (a child span when a parent is +selects that representation. With `mark_projection = "tool"`, each eligible +mark becomes a zero-duration span (a child span when parent context is available) with `openinference.span.kind = "TOOL"` and the original mark payload, category, profile, timestamp, and parent identifiers retained as attributes. High-volume `llm.chunk` marks retain exporter-native handling by @@ -98,6 +98,7 @@ default and when excluded from tool projection. Add other event names to `mark_exclude_names` for a backend to keep those marks in exporter-native form rather than visible tool children. The exclusion list affects only tool projection; it does not remove mark payload or metadata. +Set `mark_exclude_names = []` to disable the default `llm.chunk` exclusion. Each lifecycle span includes `nemo_relay.uuid` and `nemo_relay.parent_uuid` attributes. These values match ATIF `step.extra.ancestry.function_id` and diff --git a/docs/configure-plugins/observability/opentelemetry.mdx b/docs/configure-plugins/observability/opentelemetry.mdx index 335a3789b..126c689cd 100644 --- a/docs/configure-plugins/observability/opentelemetry.mdx +++ b/docs/configure-plugins/observability/opentelemetry.mdx @@ -55,7 +55,7 @@ The following table describes OpenTelemetry exporter settings: | Field | Default | Notes | |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | -| `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces span events; `tool` emits zero-duration child spans for trace-tree visibility. | +| `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces span events; `tool` emits zero-duration spans, parented as children when context is available, for trace-tree visibility. | | `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain exporter-native handling. Metadata `hook_event_name` aliases are also matched. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | @@ -77,7 +77,7 @@ The default `inherit` projection follows exporter-native handling: a mark with an active parent span is a span event, while an orphan mark is a standalone zero-duration `mark:` span. `mark_projection = "event"` explicitly selects that representation. Set `mark_projection = "tool"` only when the target -backend needs non-streaming marks represented as visible child spans. Projected +backend needs eligible marks represented as visible child spans. Projected marks use `SpanKind::Internal`, carry `nemo_relay.mark.projection = "tool"`, and retain mark UUID, parentage, category/profile, and payload attributes. High-volume `llm.chunk` marks retain @@ -86,6 +86,7 @@ exporter-native handling by default and when excluded from tool projection. Add other event names to `mark_exclude_names` for a backend to keep those marks in exporter-native form rather than visible tool children. The exclusion list affects only tool projection; it does not remove mark payload or metadata. +Set `mark_exclude_names = []` to disable the default `llm.chunk` exclusion. Each lifecycle span includes `nemo_relay.uuid` and `nemo_relay.parent_uuid` attributes. These values match ATIF `step.extra.ancestry.function_id` and From b9e6a3a7d93f61726c886a9bbff187549103844c Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 8 Jul 2026 13:31:33 -0600 Subject: [PATCH 14/14] fix(observability): gate exporter projection helpers Signed-off-by: Bryan Bednarski --- crates/core/src/observability/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/core/src/observability/mod.rs b/crates/core/src/observability/mod.rs index f2dd3a352..6884cbd96 100644 --- a/crates/core/src/observability/mod.rs +++ b/crates/core/src/observability/mod.rs @@ -53,6 +53,7 @@ pub(crate) fn default_mark_exclude_names() -> Vec { /// /// Agent hook adapters may preserve the canonical event name in metadata while /// using a generic mark name, so both representations are matched. +#[cfg(any(feature = "otel", feature = "openinference"))] pub(crate) fn mark_name_is_excluded( event: &crate::api::event::Event, excluded_names: &[String], @@ -72,6 +73,7 @@ pub(crate) fn mark_name_is_excluded( /// /// Exclusions only affect tool projection; all other modes retain their /// configured exporter-native behavior. +#[cfg(any(feature = "otel", feature = "openinference"))] pub(crate) fn effective_mark_projection( event: &crate::api::event::Event, projection: MarkProjection,