diff --git a/crates/cli/tests/coverage/session_tests.rs b/crates/cli/tests/coverage/session_tests.rs index 0a283b0bb..aa92a95e2 100644 --- a/crates/cli/tests/coverage/session_tests.rs +++ b/crates/cli/tests/coverage/session_tests.rs @@ -2896,7 +2896,7 @@ async fn hermes_post_tool_call_writes_atif_observation_with_source_call_id() { } #[tokio::test] -async fn hermes_orphan_subagent_stop_exports_readable_mark_with_lineage() { +async fn hermes_orphan_subagent_stop_does_not_create_atif_mark_step() { let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; let temp = tempfile::tempdir().unwrap(); let atif_dir = temp.path().join("atif"); @@ -2911,25 +2911,7 @@ async fn hermes_orphan_subagent_stop_exports_readable_mark_with_lineage() { let atif = read_atif_for_session(&atif_dir, "hermes-orphan"); assert!(atif["subagent_trajectories"].is_null()); let root_steps = atif["steps"].as_array().unwrap(); - assert_eq!(root_steps.len(), 1); - assert_eq!(root_steps[0]["source"], json!("system")); - assert_eq!(root_steps[0]["message"], json!("subagent_stop")); - assert_eq!( - root_steps[0]["extra"]["event_payload"]["hook_event_name"], - json!("subagent_stop") - ); - assert_eq!( - root_steps[0]["extra"]["event_payload"]["extra"]["subagent_id"], - json!("worker-1") - ); - assert_eq!( - root_steps[0]["extra"]["ancestry"]["function_name"], - json!("subagent_end_without_start") - ); - assert_eq!( - root_steps[0]["extra"]["ancestry"]["parent_name"], - json!("hermes-turn") - ); + assert!(root_steps.is_empty()); } #[tokio::test] diff --git a/crates/core/src/observability/atif.rs b/crates/core/src/observability/atif.rs index ce96140e9..96ffe8665 100644 --- a/crates/core/src/observability/atif.rs +++ b/crates/core/src/observability/atif.rs @@ -22,7 +22,7 @@ //! | LLM End | `agent` step | Response content, tool_calls promoted| //! | Tool Start | *(skipped)* | tool_calls come from LLM End instead | //! | Tool End | agent observation | Correlated by `source_call_id` | -//! | Mark (with data)| `system` step | Custom event data preserved | +//! | Mark | *(skipped)* | Point-in-time telemetry is not a step| //! | Scope Start/End | *(skipped)* | Structural events, not trajectory | //! //! The exporter serializes the full collected event stream into a single ATIF @@ -273,7 +273,10 @@ pub struct AtifStepExtra { /// Full raw LLM response payload for response-level fidelity. #[serde(skip_serializing_if = "Option::is_none")] pub llm_response: Option, - /// Full raw point-in-time event payload for mark/system steps. + /// Legacy event payload field retained for source compatibility. + /// + /// The ATIF exporter does not translate point-in-time mark events into + /// trajectory steps, so exporter-produced steps leave this field unset. #[serde(skip_serializing_if = "Option::is_none")] pub event_payload: Option, /// Per-tool callable lineage, aligned with `tool_calls`. @@ -359,11 +362,14 @@ impl AtifExporter { /// [`register_subscriber`](crate::api::subscriber::register_subscriber). /// /// # Returns - /// An [`EventSubscriberFn`] that appends each observed event to this - /// exporter's internal buffer. + /// An [`EventSubscriberFn`] that appends compatible lifecycle events to + /// this exporter's internal buffer. Point-in-time marks are ignored. pub fn subscriber(&self) -> EventSubscriberFn { let state = self.state.clone(); Arc::new(move |event: &Event| { + if matches!(event, Event::Mark(_)) { + return; + } if let Ok(mut s) = state.lock() { s.events.push(event.clone()); } @@ -2004,7 +2010,6 @@ impl StepConversionState { ("scope", Some(crate::api::event::ScopeCategory::End), Some("tool")) => { self.handle_tool_end(event, lookups) } - ("mark", _, _) => self.handle_mark(event, lookups), _ => {} } } @@ -2587,49 +2592,6 @@ impl StepConversionState { true } - fn handle_mark(&mut self, mark: &Event, lookups: &EventLookupMaps) { - if is_llm_chunk_mark(mark) { - return; - } - self.flush_observations(); - let Some(data) = mark.data() else { - return; - }; - if is_empty_mark_payload(data) { - return; - } - let extra = AtifStepExtra { - ancestry: build_ancestry(mark, &lookups.name_map), - invocation: Some(AtifInvocationInfo { - start_timestamp: None, - end_timestamp: None, - invocation_id: Some(mark.uuid().to_string()), - status: Some("completed".to_string()), - framework: Some("nemo_relay".to_string()), - }), - llm_request: None, - llm_response: None, - event_payload: Some(data.clone()), - tool_ancestry: Vec::new(), - tool_invocations: None, - }; - self.steps.push(AtifStep { - step_id: 0, - source: "system".to_string(), - message: mark_message(mark, data), - timestamp: Some(mark.timestamp().to_rfc3339()), - model_name: None, - reasoning_effort: None, - reasoning_content: None, - tool_calls: None, - observation: None, - metrics: None, - llm_call_count: None, - is_copied_context: None, - extra: serde_json::to_value(&extra).ok(), - }); - } - fn handle_subagent_start(&mut self, child: &AgentScopeNode, event: &Event) { let source_call_id = self.resolve_subagent_source_call_id(event); self.flush_observations(); @@ -3218,7 +3180,7 @@ fn is_step_event(event: &Event) -> bool { "scope", Some(crate::api::event::ScopeCategory::End), Some("tool") - ) | ("mark", _, _) + ) ) } @@ -3352,8 +3314,7 @@ fn events_to_steps_for_agent( /// step with multiple results /// 4. Tool End observation results are correlated with the preceding LLM End's /// promoted tool_calls by function name → `source_call_id`. -/// 5. Mark events → system steps if they carry data. -/// 6. Scope Start/End → skipped. +/// 5. Mark and other Scope Start/End events → skipped. fn events_to_steps(events: &[&Event]) -> Vec { let mut sorted: Vec<&Event> = events.to_vec(); sorted.sort_by_key(|e| *e.timestamp()); @@ -3367,41 +3328,6 @@ fn events_to_steps(events: &[&Event]) -> Vec { state.finish() } -fn is_empty_mark_payload(data: &Json) -> bool { - data.is_null() || data.as_object().is_some_and(|object| object.is_empty()) -} - -fn is_llm_chunk_mark(mark: &Event) -> bool { - mark.name() == "llm.chunk" - || mark - .metadata() - .and_then(Json::as_object) - .and_then(|metadata| metadata.get("hook_event_name")) - .and_then(Json::as_str) - == Some("llm.chunk") -} - -// A runtime mark is point-in-time telemetry rather than a scoped call with start/end events. Agent -// hook adapters use marks for lifecycle notifications that do not map to first-class ATIF step -// types, for example hook-only status updates or synthetic fallback events. The ATIF step message -// stays schema-compatible while the original payload is preserved in `Step.extra.event_payload`. -fn mark_message(mark: &Event, _data: &Json) -> Json { - Json::String(mark_hook_event_name(mark).unwrap_or_default()) -} - -// Prefer the adapter-provided hook name because the runtime mark name may be a generic bucket such -// as `hook_mark` or a synthetic fallback like `subagent_end_without_start`. Falling back to the mark -// name keeps non-hook marks readable without making this exporter depend on any one agent adapter. -fn mark_hook_event_name(mark: &Event) -> Option { - mark.metadata() - .and_then(Json::as_object) - .and_then(|metadata| metadata.get("hook_event_name")) - .and_then(Json::as_str) - .filter(|name| !name.is_empty()) - .map(ToOwned::to_owned) - .or_else(|| Some(mark.name().to_string()).filter(|name| !name.is_empty())) -} - fn is_start_event(event: &Event) -> bool { event.scope_category() == Some(crate::api::event::ScopeCategory::Start) } diff --git a/crates/core/src/observability/mod.rs b/crates/core/src/observability/mod.rs index fd35def8d..6884cbd96 100644 --- a/crates/core/src/observability/mod.rs +++ b/crates/core/src/observability/mod.rs @@ -4,6 +4,7 @@ //! Optional observability integrations for NeMo Relay Core. use crate::api::event::EventNormalizationExt; +use serde::{Deserialize, Serialize}; #[cfg(test)] use std::sync::Mutex; @@ -22,6 +23,69 @@ pub mod openinference; pub mod otel; pub mod plugin_component; +/// Export representation for point-in-time mark events. +/// +/// Marks remain canonical ATOF events regardless of this setting. Exporters +/// apply the selected projection only when translating those events into a +/// downstream trace format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum MarkProjection { + /// Use each exporter’s native handling for marks. + #[default] + Inherit, + /// Force marks into exporter-native trace span events. + Event, + /// Render non-excluded marks as zero-duration trace child spans so + /// trace-tree consumers can display them directly. High-volume + /// `llm.chunk` marks remain exporter-native events. + Tool, +} + +/// Default mark names excluded from tool projection because they are emitted +/// at high volume and are better represented as exporter-native events. +pub(crate) fn default_mark_exclude_names() -> Vec { + vec!["llm.chunk".to_string()] +} + +/// Returns whether a mark matches a configured projection exclusion. +/// +/// Agent hook adapters may preserve the canonical event name in metadata while +/// using a generic mark name, so both representations are matched. +#[cfg(any(feature = "otel", feature = "openinference"))] +pub(crate) fn mark_name_is_excluded( + event: &crate::api::event::Event, + excluded_names: &[String], +) -> bool { + excluded_names.iter().any(|name| { + event.name() == name + || event + .metadata() + .and_then(crate::json::Json::as_object) + .and_then(|metadata| metadata.get("hook_event_name")) + .and_then(crate::json::Json::as_str) + == Some(name.as_str()) + }) +} + +/// Resolves a configured mark projection for one event. +/// +/// Exclusions only affect tool projection; all other modes retain their +/// configured exporter-native behavior. +#[cfg(any(feature = "otel", feature = "openinference"))] +pub(crate) fn effective_mark_projection( + event: &crate::api::event::Event, + projection: MarkProjection, + excluded_names: &[String], +) -> MarkProjection { + if projection == MarkProjection::Tool && mark_name_is_excluded(event, excluded_names) { + MarkProjection::Inherit + } else { + projection + } +} + #[cfg(all(test, feature = "otel", feature = "openinference"))] #[path = "../../tests/unit/observability/exporter_parity_tests.rs"] mod exporter_parity_tests; diff --git a/crates/core/src/observability/openinference.rs b/crates/core/src/observability/openinference.rs index 649460c96..15676277d 100644 --- a/crates/core/src/observability/openinference.rs +++ b/crates/core/src/observability/openinference.rs @@ -7,7 +7,7 @@ //! //! - scope/tool/LLM `Start` events open spans //! - matching `End` events close spans -//! - `Mark` events become span events on the active parent span when possible +//! - `Mark` events become span events by default, with an optional OpenInference tool projection //! - orphan marks fall back to zero-duration spans so they still reach OTLP //! //! The public API is intentionally small: @@ -21,6 +21,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use super::{ + MarkProjection, default_mark_exclude_names, effective_mark_projection, estimate_cost_for_response_or_model, estimate_cost_for_response_or_requested_model, manual, merge_usage, model_name_for_llm_event, }; @@ -101,6 +102,8 @@ pub struct OpenInferenceConfig { service_namespace: Option, service_version: Option, instrumentation_scope: String, + mark_projection: MarkProjection, + mark_exclude_names: Vec, timeout: Duration, transport: OtlpTransport, } @@ -115,6 +118,8 @@ impl Default for OpenInferenceConfig { service_namespace: None, service_version: None, instrumentation_scope: "nemo-relay-openinference".to_string(), + mark_projection: MarkProjection::default(), + mark_exclude_names: default_mark_exclude_names(), timeout: Duration::from_secs(3), transport: OtlpTransport::HttpBinary, } @@ -184,6 +189,24 @@ impl OpenInferenceConfig { self.instrumentation_scope = scope.into(); self } + + /// Selects how point-in-time marks are represented in exported traces. + pub fn with_mark_projection(mut self, mark_projection: MarkProjection) -> Self { + self.mark_projection = mark_projection; + self + } + + /// Excludes named marks from tool projection while preserving their native + /// event representation. The default excludes high-volume `llm.chunk` + /// marks. + pub fn with_mark_exclude_names(mut self, names: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.mark_exclude_names = names.into_iter().map(Into::into).collect(); + self + } } /// OpenInference-backed NeMo Relay subscriber. @@ -209,6 +232,8 @@ impl OpenInferenceSubscriber { Ok(Self::from_tracer_provider_with_scope( provider, config.instrumentation_scope, + config.mark_projection, + config.mark_exclude_names, )) } @@ -217,17 +242,61 @@ impl OpenInferenceSubscriber { provider: SdkTracerProvider, instrumentation_scope: impl Into, ) -> Self { - Self::from_tracer_provider_with_scope(provider, instrumentation_scope.into()) + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + MarkProjection::default(), + default_mark_exclude_names(), + ) + } + + /// Builds a subscriber from a tracer provider with an explicit mark projection. + pub fn from_tracer_provider_with_mark_projection( + provider: SdkTracerProvider, + instrumentation_scope: impl Into, + mark_projection: MarkProjection, + ) -> Self { + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + mark_projection, + default_mark_exclude_names(), + ) + } + + /// Builds a subscriber with explicit mark projection and exclusion names. + pub fn from_tracer_provider_with_mark_projection_and_exclusions( + provider: SdkTracerProvider, + instrumentation_scope: impl Into, + mark_projection: MarkProjection, + mark_exclude_names: I, + ) -> Self + where + I: IntoIterator, + S: Into, + { + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + mark_projection, + mark_exclude_names.into_iter().map(Into::into).collect(), + ) } fn from_tracer_provider_with_scope( provider: SdkTracerProvider, instrumentation_scope: String, + mark_projection: MarkProjection, + mark_exclude_names: Vec, ) -> Self { - let processor = Arc::new(Mutex::new(OpenInferenceEventProcessor::new( - provider, - instrumentation_scope, - ))); + let processor = Arc::new(Mutex::new( + OpenInferenceEventProcessor::new_with_mark_projection_and_exclusions( + provider, + instrumentation_scope, + mark_projection, + mark_exclude_names, + ), + )); let processor_for_callback = Arc::clone(&processor); let subscriber: EventSubscriberFn = Arc::new(move |event: &Event| { let Ok(mut guard) = processor_for_callback.lock() else { @@ -381,10 +450,36 @@ struct OpenInferenceEventProcessor { completed_span_order: VecDeque, provider: SdkTracerProvider, tracer: SdkTracer, + mark_projection: MarkProjection, + mark_exclude_names: Vec, } impl OpenInferenceEventProcessor { + #[cfg(test)] fn new(provider: SdkTracerProvider, instrumentation_scope: String) -> Self { + Self::new_with_mark_projection(provider, instrumentation_scope, MarkProjection::default()) + } + + #[cfg(test)] + fn new_with_mark_projection( + provider: SdkTracerProvider, + instrumentation_scope: String, + mark_projection: MarkProjection, + ) -> Self { + Self::new_with_mark_projection_and_exclusions( + provider, + instrumentation_scope, + mark_projection, + default_mark_exclude_names(), + ) + } + + fn new_with_mark_projection_and_exclusions( + provider: SdkTracerProvider, + instrumentation_scope: String, + mark_projection: MarkProjection, + mark_exclude_names: Vec, + ) -> Self { let tracer = provider.tracer(instrumentation_scope); Self { active_spans: HashMap::new(), @@ -392,6 +487,8 @@ impl OpenInferenceEventProcessor { completed_span_order: VecDeque::new(), provider, tracer, + mark_projection, + mark_exclude_names, } } @@ -442,6 +539,12 @@ impl OpenInferenceEventProcessor { } fn process_mark(&mut self, event: &Event) { + if effective_mark_projection(event, self.mark_projection, &self.mark_exclude_names) + == MarkProjection::Tool + { + self.process_mark_as_tool(event); + return; + } let mark_name = event.name().to_string(); let timestamp = to_system_time(*event.timestamp()); let attributes = mark_attributes(event); @@ -469,6 +572,29 @@ impl OpenInferenceEventProcessor { span.end_with_timestamp(timestamp); } + fn process_mark_as_tool(&mut self, event: &Event) { + let timestamp = to_system_time(*event.timestamp()); + let orphan = self.find_parent_span(event).is_none(); + let mut attributes = mark_attributes(event); + attributes.push(KeyValue::new("nemo_relay.mark.projection", "tool")); + attributes.push(KeyValue::new( + oi::OPENINFERENCE_SPAN_KIND, + OpenInferenceSpanKind::Tool, + )); + if orphan { + attributes.push(KeyValue::new("nemo_relay.mark.orphan", true)); + } + + let mut span = self + .tracer + .span_builder(format!("mark:{}", event.name())) + .with_kind(SpanKind::Internal) + .with_start_time(timestamp) + .start_with_context(&self.tracer, &self.parent_context(event)); + span.set_attributes(attributes); + span.end_with_timestamp(timestamp); + } + fn parent_context(&self, event: &Event) -> Context { if let Some(active_span) = self.find_parent_span(event) { return Context::new().with_remote_span_context(active_span.span_context.clone()); @@ -1108,6 +1234,17 @@ fn mark_attributes(event: &Event) -> Vec { "nemo_relay.mark.metadata_json", event.metadata(), ); + if let Some(category) = event.category() { + attributes.push(KeyValue::new( + "nemo_relay.mark.category", + category.as_str().to_string(), + )); + } + push_serialized( + &mut attributes, + "nemo_relay.mark.category_profile_json", + event.category_profile(), + ); attributes } diff --git a/crates/core/src/observability/otel.rs b/crates/core/src/observability/otel.rs index 474af253b..727b60a79 100644 --- a/crates/core/src/observability/otel.rs +++ b/crates/core/src/observability/otel.rs @@ -7,7 +7,7 @@ //! //! - scope/tool/LLM `Start` events open spans //! - matching `End` events close spans -//! - `Mark` events become span events on the active parent span when possible +//! - `Mark` events become span events by default, with an optional visible child-span projection //! - orphan marks fall back to zero-duration spans so they still reach OTLP //! //! The public API is intentionally small: @@ -21,6 +21,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use super::{ + MarkProjection, default_mark_exclude_names, effective_mark_projection, estimate_cost_for_response_or_model, estimate_cost_for_response_or_requested_model, manual, model_name_for_llm_event, }; @@ -95,6 +96,8 @@ pub struct OpenTelemetryConfig { service_namespace: Option, service_version: Option, instrumentation_scope: String, + mark_projection: MarkProjection, + mark_exclude_names: Vec, timeout: Duration, transport: OtlpTransport, } @@ -109,6 +112,8 @@ impl Default for OpenTelemetryConfig { service_namespace: None, service_version: None, instrumentation_scope: "nemo-relay-otel".to_string(), + mark_projection: MarkProjection::default(), + mark_exclude_names: default_mark_exclude_names(), timeout: Duration::from_secs(3), transport: OtlpTransport::HttpBinary, } @@ -179,6 +184,24 @@ impl OpenTelemetryConfig { self.instrumentation_scope = scope.into(); self } + + /// Selects how point-in-time marks are represented in exported traces. + pub fn with_mark_projection(mut self, mark_projection: MarkProjection) -> Self { + self.mark_projection = mark_projection; + self + } + + /// Excludes named marks from tool projection while preserving their native + /// event representation. The default excludes high-volume `llm.chunk` + /// marks. + pub fn with_mark_exclude_names(mut self, names: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.mark_exclude_names = names.into_iter().map(Into::into).collect(); + self + } } /// OpenTelemetry-backed NeMo Relay subscriber. @@ -204,6 +227,8 @@ impl OpenTelemetrySubscriber { Ok(Self::from_tracer_provider_with_scope( provider, config.instrumentation_scope, + config.mark_projection, + config.mark_exclude_names, )) } @@ -212,17 +237,61 @@ impl OpenTelemetrySubscriber { provider: SdkTracerProvider, instrumentation_scope: impl Into, ) -> Self { - Self::from_tracer_provider_with_scope(provider, instrumentation_scope.into()) + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + MarkProjection::default(), + default_mark_exclude_names(), + ) + } + + /// Builds a subscriber from a tracer provider with an explicit mark projection. + pub fn from_tracer_provider_with_mark_projection( + provider: SdkTracerProvider, + instrumentation_scope: impl Into, + mark_projection: MarkProjection, + ) -> Self { + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + mark_projection, + default_mark_exclude_names(), + ) + } + + /// Builds a subscriber with explicit mark projection and exclusion names. + pub fn from_tracer_provider_with_mark_projection_and_exclusions( + provider: SdkTracerProvider, + instrumentation_scope: impl Into, + mark_projection: MarkProjection, + mark_exclude_names: I, + ) -> Self + where + I: IntoIterator, + S: Into, + { + Self::from_tracer_provider_with_scope( + provider, + instrumentation_scope.into(), + mark_projection, + mark_exclude_names.into_iter().map(Into::into).collect(), + ) } fn from_tracer_provider_with_scope( provider: SdkTracerProvider, instrumentation_scope: String, + mark_projection: MarkProjection, + mark_exclude_names: Vec, ) -> Self { - let processor = Arc::new(Mutex::new(OtelEventProcessor::new( - provider, - instrumentation_scope, - ))); + let processor = Arc::new(Mutex::new( + OtelEventProcessor::new_with_mark_projection_and_exclusions( + provider, + instrumentation_scope, + mark_projection, + mark_exclude_names, + ), + )); let processor_for_callback = Arc::clone(&processor); let subscriber: EventSubscriberFn = Arc::new(move |event: &Event| { let Ok(mut guard) = processor_for_callback.lock() else { @@ -375,10 +444,36 @@ struct OtelEventProcessor { completed_span_order: VecDeque, provider: SdkTracerProvider, tracer: SdkTracer, + mark_projection: MarkProjection, + mark_exclude_names: Vec, } impl OtelEventProcessor { + #[cfg(test)] fn new(provider: SdkTracerProvider, instrumentation_scope: String) -> Self { + Self::new_with_mark_projection(provider, instrumentation_scope, MarkProjection::default()) + } + + #[cfg(test)] + fn new_with_mark_projection( + provider: SdkTracerProvider, + instrumentation_scope: String, + mark_projection: MarkProjection, + ) -> Self { + Self::new_with_mark_projection_and_exclusions( + provider, + instrumentation_scope, + mark_projection, + default_mark_exclude_names(), + ) + } + + fn new_with_mark_projection_and_exclusions( + provider: SdkTracerProvider, + instrumentation_scope: String, + mark_projection: MarkProjection, + mark_exclude_names: Vec, + ) -> Self { let tracer = provider.tracer(instrumentation_scope); Self { active_spans: HashMap::new(), @@ -386,6 +481,8 @@ impl OtelEventProcessor { completed_span_order: VecDeque::new(), provider, tracer, + mark_projection, + mark_exclude_names, } } @@ -437,6 +534,12 @@ impl OtelEventProcessor { } fn process_mark(&mut self, event: &Event) { + if effective_mark_projection(event, self.mark_projection, &self.mark_exclude_names) + == MarkProjection::Tool + { + self.process_mark_as_tool(event); + return; + } let mark_name = event.name().to_string(); let timestamp = to_system_time(*event.timestamp()); let attributes = mark_attributes(event); @@ -460,6 +563,26 @@ impl OtelEventProcessor { span.end_with_timestamp(timestamp); } + fn process_mark_as_tool(&mut self, event: &Event) { + let timestamp = to_system_time(*event.timestamp()); + let orphan = self.find_parent_span(event).is_none(); + let mut attributes = mark_attributes(event); + attributes.push(KeyValue::new("nemo_relay.mark.projection", "tool")); + attributes.push(KeyValue::new("nemo_relay.scope_type", "tool")); + if orphan { + attributes.push(KeyValue::new("nemo_relay.mark.orphan", true)); + } + + let mut span = self + .tracer + .span_builder(format!("mark:{}", event.name())) + .with_kind(SpanKind::Internal) + .with_start_time(timestamp) + .start_with_context(&self.tracer, &self.parent_context(event)); + span.set_attributes(attributes); + span.end_with_timestamp(timestamp); + } + fn parent_context(&self, event: &Event) -> Context { if let Some(active_span) = self.find_parent_span(event) { return Context::new().with_remote_span_context(active_span.span_context.clone()); @@ -645,6 +768,17 @@ fn mark_attributes(event: &Event) -> Vec { "nemo_relay.mark.metadata_json", event.metadata(), ); + if let Some(category) = event.category() { + attributes.push(KeyValue::new( + "nemo_relay.mark.category", + category.as_str().to_string(), + )); + } + push_serialized( + &mut attributes, + "nemo_relay.mark.category_profile_json", + event.category_profile(), + ); attributes } diff --git a/crates/core/src/observability/plugin_component.rs b/crates/core/src/observability/plugin_component.rs index ff8e62197..8481d0640 100644 --- a/crates/core/src/observability/plugin_component.rs +++ b/crates/core/src/observability/plugin_component.rs @@ -51,6 +51,7 @@ use crate::observability::openinference::{ use crate::observability::otel::{ OpenTelemetryConfig as CoreOpenTelemetryConfig, OpenTelemetrySubscriber, }; +use crate::observability::{MarkProjection, default_mark_exclude_names}; use crate::plugin::{ ConfigDiagnostic, ConfigPolicy, DiagnosticLevel, Plugin, PluginComponentSpec, PluginError, PluginRegistration, PluginRegistrationContext, Result as PluginResult, UnsupportedBehavior, @@ -374,6 +375,13 @@ pub struct OtlpSectionConfig { /// Whether the subscriber is active. #[serde(default)] pub enabled: bool, + /// Representation used for mark events: `inherit`, `event`, or `tool`. + #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "mark_projection_schema"))] + pub mark_projection: MarkProjection, + /// Mark names excluded from tool projection. Defaults to `llm.chunk`. + #[serde(default = "default_mark_exclude_names")] + pub mark_exclude_names: Vec, /// OTLP transport: `http_binary` or `grpc`. #[serde(default = "default_otlp_transport")] #[cfg_attr(feature = "schema", schemars(schema_with = "otlp_transport_schema"))] @@ -408,6 +416,8 @@ impl Default for OtlpSectionConfig { fn default() -> Self { Self { enabled: false, + mark_projection: MarkProjection::default(), + mark_exclude_names: default_mark_exclude_names(), transport: default_otlp_transport(), endpoint: None, headers: HashMap::new(), @@ -487,6 +497,8 @@ crate::editor_config! { crate::editor_config! { impl OtlpSectionConfig { enabled => { label: "enabled", kind: Boolean }, + mark_projection => { label: "mark_projection", kind: Enum, values: ["inherit", "event", "tool"] }, + mark_exclude_names => { label: "mark_exclude_names", kind: Json }, transport => { label: "transport", kind: Enum, values: ["http_binary", "grpc"] }, endpoint => { label: "endpoint", kind: String, optional: true }, headers => { label: "headers", kind: StringMap }, @@ -581,6 +593,13 @@ fn otlp_transport_schema( string_enum_schema(generator, &["http_binary", "grpc"], Some("http_binary")) } +#[cfg(feature = "schema")] +fn mark_projection_schema( + generator: &mut schemars::r#gen::SchemaGenerator, +) -> schemars::schema::Schema { + string_enum_schema(generator, &["inherit", "event", "tool"], Some("inherit")) +} + #[cfg(feature = "schema")] fn string_enum_schema( generator: &mut schemars::r#gen::SchemaGenerator, @@ -1347,6 +1366,9 @@ fn build_otel_config(section: OtlpSectionConfig) -> PluginResult PluginResult::new())); + let captured = Arc::clone(&events); + register_subscriber( + "managed_tool_projection_events", + Arc::new(move |event| captured.lock().unwrap().push(event.clone())), + ) + .unwrap(); + + let atif = AtifExporter::new( + "managed-tool-projection".to_string(), + AtifAgentInfo { + name: "test-agent".to_string(), + version: "1.0.0".to_string(), + model_name: None, + tool_definitions: None, + extra: None, + }, + ); + register_subscriber("managed_tool_projection_atif", atif.subscriber()).unwrap(); + + let otel_exporter = InMemorySpanExporterBuilder::new().build(); + let otel_provider = SdkTracerProvider::builder() + .with_simple_exporter(otel_exporter.clone()) + .build(); + let otel = OpenTelemetrySubscriber::from_tracer_provider_with_mark_projection( + otel_provider, + "managed-tool-projection-otel", + MarkProjection::Tool, + ); + register_subscriber("managed_tool_projection_otel", otel.subscriber()).unwrap(); + + let openinference_exporter = InMemorySpanExporterBuilder::new().build(); + let openinference_provider = SdkTracerProvider::builder() + .with_simple_exporter(openinference_exporter.clone()) + .build(); + let openinference = OpenInferenceSubscriber::from_tracer_provider_with_mark_projection( + openinference_provider, + "managed-tool-projection-openinference", + MarkProjection::Tool, + ); + register_subscriber( + "managed_tool_projection_openinference", + openinference.subscriber(), + ) + .unwrap(); + + register_tool_execution_intercept( + "managed_tool_projection_intercept", + 1, + Arc::new(|_name, args, next| { + Box::pin(async move { + let result = next(args).await?; + Ok( + ToolExecutionInterceptOutcome::new(result).with_pending_mark( + PendingMarkSpec::builder() + .name("plugin.output_compacted") + .category(EventCategory::custom()) + .category_profile( + CategoryProfile::builder() + .subtype("example.compaction") + .build(), + ) + .data(json!({"saved_tokens": 12})) + .metadata(json!({"source": "test"})) + .build(), + ), + ) + }) + }), + ) + .unwrap(); + + let result = tool_call_execute( + nemo_relay::api::tool::ToolCallExecuteParams::builder() + .name("managed-tool") + .args(json!({"value": 42})) + .func(Arc::new(|args| Box::pin(async move { Ok(args) }))) + .build(), + ) + .await + .unwrap(); + assert_eq!(result, json!({"value": 42})); + + flush_subscribers().unwrap(); + let captured = events.lock().unwrap(); + let tool_end_index = captured + .iter() + .position(|event| { + event.name() == "managed-tool" && event.scope_category() == Some(ScopeCategory::End) + }) + .unwrap(); + let mark_index = captured + .iter() + .position(|event| event.name() == "plugin.output_compacted") + .unwrap(); + assert!(tool_end_index < mark_index); + let tool_end = &captured[tool_end_index]; + let mark = &captured[mark_index]; + assert_eq!(mark.parent_uuid(), Some(tool_end.uuid())); + assert!(mark.timestamp() > tool_end.timestamp()); + drop(captured); + + let trajectory = atif.export().unwrap(); + assert!(trajectory.steps.iter().all(|step| { + !step.tool_calls.as_deref().is_some_and(|calls| { + calls + .iter() + .any(|call| call.function_name == "plugin.output_compacted") + }) + })); + + otel.force_flush().unwrap(); + let otel_spans = otel_exporter.get_finished_spans().unwrap(); + let otel_tool = otel_spans + .iter() + .find(|span| span.name.as_ref() == "managed-tool") + .unwrap(); + let otel_mark = otel_spans + .iter() + .find(|span| span.name.as_ref() == "mark:plugin.output_compacted") + .unwrap(); + assert_eq!(otel_mark.parent_span_id, otel_tool.span_context.span_id()); + assert!(otel_mark.start_time > otel_tool.end_time); + assert!(otel_mark.attributes.iter().any(|attribute| { + attribute.key.as_str() == "nemo_relay.mark.projection" + && attribute.value.to_string() == "tool" + })); + for (key, value) in [ + ("nemo_relay.mark.category", "custom"), + ( + "nemo_relay.mark.category_profile_json", + "{\"subtype\":\"example.compaction\"}", + ), + ("nemo_relay.mark.metadata_json", "{\"source\":\"test\"}"), + ] { + assert!(otel_mark.attributes.iter().any(|attribute| { + attribute.key.as_str() == key && attribute.value.to_string() == value + })); + } + + openinference.force_flush().unwrap(); + let openinference_spans = openinference_exporter.get_finished_spans().unwrap(); + let openinference_tool = openinference_spans + .iter() + .find(|span| span.name.as_ref() == "managed-tool") + .unwrap(); + let openinference_mark = openinference_spans + .iter() + .find(|span| span.name.as_ref() == "mark:plugin.output_compacted") + .unwrap(); + assert_eq!( + openinference_mark.parent_span_id, + openinference_tool.span_context.span_id() + ); + assert!(openinference_mark.start_time > openinference_tool.end_time); + assert!(openinference_mark.attributes.iter().any(|attribute| { + attribute.key.as_str() == "openinference.span.kind" && attribute.value.to_string() == "TOOL" + })); + for (key, value) in [ + ("nemo_relay.mark.category", "custom"), + ( + "nemo_relay.mark.category_profile_json", + "{\"subtype\":\"example.compaction\"}", + ), + ("nemo_relay.mark.metadata_json", "{\"source\":\"test\"}"), + ] { + assert!(openinference_mark.attributes.iter().any(|attribute| { + attribute.key.as_str() == key && attribute.value.to_string() == value + })); + } + + deregister_tool_execution_intercept("managed_tool_projection_intercept").unwrap(); + deregister_subscriber("managed_tool_projection_events").unwrap(); + deregister_subscriber("managed_tool_projection_atif").unwrap(); + deregister_subscriber("managed_tool_projection_otel").unwrap(); + deregister_subscriber("managed_tool_projection_openinference").unwrap(); +} + #[tokio::test] async fn test_tool_execution_error_discards_downstream_pending_marks() { let _lock = TEST_MUTEX.lock().unwrap(); diff --git a/crates/core/tests/unit/atif_tests.rs b/crates/core/tests/unit/atif_tests.rs index 78e3c3b89..6a994b27e 100644 --- a/crates/core/tests/unit/atif_tests.rs +++ b/crates/core/tests/unit/atif_tests.rs @@ -1789,7 +1789,7 @@ fn test_exporter_openclaw_placeholder_replay_preserves_empty_user_step_and_raw_r } #[test] -fn test_exporter_openclaw_timing_marks_become_system_steps_with_payloads() { +fn test_exporter_ignores_openclaw_timing_marks() { let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); let base = base_timestamp(); @@ -1821,40 +1821,14 @@ fn test_exporter_openclaw_timing_marks_become_system_steps_with_payloads() { set_event_timestamp(&mut ambiguous, base); set_event_timestamp(&mut unpaired, base + chrono::Duration::milliseconds(1)); - { - let mut state = exporter.state.lock().unwrap(); - state.events.extend([ambiguous, unpaired]); - } + let subscriber = exporter.subscriber(); + subscriber(&ambiguous); + subscriber(&unpaired); + assert!(exporter.state.lock().unwrap().events.is_empty()); let trajectory = exporter.export().unwrap(); assert_atif_v17_shape(&trajectory); - assert_eq!(trajectory.steps.len(), 2); - - let ambiguous_step = &trajectory.steps[0]; - assert_eq!(ambiguous_step.source, "system"); - assert_eq!( - ambiguous_step.message, - json!("openclaw.model_call_timing_ambiguous") - ); - let ambiguous_extra: AtifStepExtra = - serde_json::from_value(ambiguous_step.extra.clone().unwrap()).unwrap(); - assert_eq!( - ambiguous_extra.event_payload.as_ref(), - Some(&ambiguous_payload) - ); - - let unpaired_step = &trajectory.steps[1]; - assert_eq!(unpaired_step.source, "system"); - assert_eq!( - unpaired_step.message, - json!("openclaw.model_call_timing_unpaired") - ); - let unpaired_extra: AtifStepExtra = - serde_json::from_value(unpaired_step.extra.clone().unwrap()).unwrap(); - assert_eq!( - unpaired_extra.event_payload.as_ref(), - Some(&unpaired_payload) - ); + assert!(trajectory.steps.is_empty()); } #[test] @@ -2619,16 +2593,15 @@ fn test_exporter_tool_call_id_linking() { } #[test] -fn test_exporter_mark_steps_include_hook_name_and_ancestry() { +fn test_exporter_ignores_marks_with_hook_metadata() { let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); let agent_uuid = Uuid::now_v7(); - let mark_uuid = Uuid::now_v7(); let agent_start = event_builder(agent_uuid, EventType::Start) .name("hermes") .scope_type(ScopeType::Agent) .build(); - let mark = event_builder(mark_uuid, EventType::Mark) + let mark = event_builder(Uuid::now_v7(), EventType::Mark) .name("subagent_end_without_start") .parent_uuid(agent_uuid) .data(json!({ @@ -2649,25 +2622,7 @@ fn test_exporter_mark_steps_include_hook_name_and_ancestry() { } let trajectory = exporter.export().unwrap(); - assert_eq!(trajectory.steps.len(), 1); - - let step = &trajectory.steps[0]; - assert_eq!(step.source, "system"); - assert_eq!(step.message, json!("subagent_stop")); - - let extra: AtifStepExtra = serde_json::from_value(step.extra.clone().unwrap()).unwrap(); - assert_eq!( - extra.event_payload.as_ref().unwrap()["extra"]["subagent_id"], - json!("worker-1") - ); - assert_eq!(extra.ancestry.function_id, mark_uuid.to_string()); - assert_eq!(extra.ancestry.function_name, "subagent_end_without_start"); - assert_eq!(extra.ancestry.parent_id, Some(agent_uuid.to_string())); - assert_eq!(extra.ancestry.parent_name, Some("hermes".to_string())); - assert_eq!( - extra.invocation.as_ref().unwrap().invocation_id, - Some(mark_uuid.to_string()) - ); + assert!(trajectory.steps.is_empty()); } #[test] @@ -2833,10 +2788,11 @@ fn test_exporter_attaches_subagent_ref_to_delegating_tool_observation() { "tool_call_id": "call_delegate" })) .build(); - let mut child_mark = event_builder(child_mark_uuid, EventType::Mark) - .name("worker-started") + let mut child_mark = event_builder(child_mark_uuid, EventType::End) + .name("worker-llm") + .scope_type(ScopeType::Llm) .parent_uuid(child_uuid) - .data(json!({"status": "started"})) + .output(json!({"content": "worker started"})) .build(); let mut child_end = event_builder(child_uuid, EventType::End) .name("worker-agent") @@ -2934,10 +2890,11 @@ fn test_exporter_synthesizes_tool_call_for_active_subagent_dispatch() { .parent_uuid(root_uuid) .metadata(json!({"session_id": "child-session"})) .build(); - let mut child_mark = event_builder(child_mark_uuid, EventType::Mark) - .name("worker-started") + let mut child_mark = event_builder(child_mark_uuid, EventType::End) + .name("worker-llm") + .scope_type(ScopeType::Llm) .parent_uuid(child_uuid) - .data(json!({"status": "started"})) + .output(json!({"content": "worker started"})) .build(); let mut child_end = event_builder(child_uuid, EventType::End) .name("worker-agent") @@ -3257,11 +3214,9 @@ fn test_exporter_renumbers_after_pruning_empty_subagent_ref_step() { let trajectory = exporter.export().unwrap(); assert_atif_v17_shape(&trajectory); - assert_eq!(trajectory.steps.len(), 2); + assert_eq!(trajectory.steps.len(), 1); assert_eq!(trajectory.steps[0].step_id, 1); assert_eq!(trajectory.steps[0].source, "agent"); - assert_eq!(trajectory.steps[1].step_id, 2); - assert_eq!(trajectory.steps[1].source, "system"); assert!(trajectory.subagent_trajectories.is_none()); let serialized = serde_json::to_value(&trajectory).unwrap(); assert!(!serialized.to_string().contains("subagent_trajectory_ref")); @@ -3291,10 +3246,11 @@ fn test_exporter_embeds_recursive_subagent_trajectories() { .parent_uuid(child_uuid) .metadata(json!({"session_id": "grandchild-session"})) .build(); - let mut grandchild_mark = event_builder(Uuid::now_v7(), EventType::Mark) - .name("deep-note") + let mut grandchild_mark = event_builder(Uuid::now_v7(), EventType::End) + .name("deep-llm") + .scope_type(ScopeType::Llm) .parent_uuid(grandchild_uuid) - .data(json!({"status": "ok"})) + .output(json!({"content": "deep-note"})) .build(); let mut grandchild_end = event_builder(grandchild_uuid, EventType::End) .name("deep-worker") @@ -3357,13 +3313,10 @@ fn test_exporter_embeds_recursive_subagent_trajectories() { assert_eq!(grandchild.steps.len(), 1); assert_eq!(grandchild.steps[0].step_id, 1); assert_eq!(grandchild.steps[0].message, json!("deep-note")); - let extra: AtifStepExtra = - serde_json::from_value(grandchild.steps[0].extra.clone().unwrap()).unwrap(); - assert_eq!(extra.event_payload.as_ref().unwrap()["status"], json!("ok")); } #[test] -fn test_exporter_skips_empty_mark_payloads() { +fn test_exporter_skips_all_mark_payloads() { let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); { @@ -3374,6 +3327,12 @@ fn test_exporter_skips_empty_mark_payloads() { .data(json!({})) .build(), ); + state.events.push( + event_builder(Uuid::now_v7(), EventType::Mark) + .name("populated") + .data(json!({"status": "ok"})) + .build(), + ); state.events.push( event_builder(Uuid::now_v7(), EventType::Mark) .name("empty-null") @@ -3391,7 +3350,7 @@ fn test_exporter_skips_empty_mark_payloads() { } #[test] -fn test_exporter_skips_llm_chunk_marks() { +fn test_exporter_skips_marks_regardless_of_name() { let exporter = AtifExporter::new("session-1".to_string(), make_agent_info()); { @@ -3419,8 +3378,7 @@ fn test_exporter_skips_llm_chunk_marks() { let trajectory = exporter.export().unwrap(); - assert_eq!(trajectory.steps.len(), 1); - assert_eq!(trajectory.steps[0].message, json!("agent.status")); + assert!(trajectory.steps.is_empty()); } #[test] @@ -4001,8 +3959,9 @@ fn test_exporter_clear() { { let mut state = exporter.state.lock().unwrap(); state.events.push( - event_builder(Uuid::now_v7(), EventType::Mark) - .data(json!("test")) + event_builder(Uuid::now_v7(), EventType::End) + .scope_type(ScopeType::Llm) + .output(json!("test")) .build(), ); } diff --git a/crates/core/tests/unit/observability/openinference_tests.rs b/crates/core/tests/unit/observability/openinference_tests.rs index f026f08aa..f9581cf6b 100644 --- a/crates/core/tests/unit/observability/openinference_tests.rs +++ b/crates/core/tests/unit/observability/openinference_tests.rs @@ -466,6 +466,8 @@ fn config_defaults_and_builder_overrides_are_applied() { .with_service_namespace("agents") .with_service_version("1.2.3") .with_instrumentation_scope("demo-scope") + .with_mark_projection(MarkProjection::Tool) + .with_mark_exclude_names(["notification", "hook_mark"]) .with_timeout(Duration::from_millis(1250)); assert_eq!(config.transport, OtlpTransport::HttpBinary); @@ -485,12 +487,16 @@ fn config_defaults_and_builder_overrides_are_applied() { assert_eq!(config.service_namespace.as_deref(), Some("agents")); assert_eq!(config.service_version.as_deref(), Some("1.2.3")); assert_eq!(config.instrumentation_scope, "demo-scope"); + assert_eq!(config.mark_projection, MarkProjection::Tool); + assert_eq!(config.mark_exclude_names, vec!["notification", "hook_mark"]); assert_eq!(config.timeout, Duration::from_millis(1250)); let defaults = OpenInferenceConfig::default(); assert_eq!(defaults.transport, OtlpTransport::HttpBinary); assert_eq!(defaults.service_name, "nemo-relay"); assert_eq!(defaults.instrumentation_scope, "nemo-relay-openinference"); + assert_eq!(defaults.mark_projection, MarkProjection::Inherit); + assert_eq!(defaults.mark_exclude_names, vec!["llm.chunk"]); assert_eq!(defaults.timeout, Duration::from_secs(3)); assert!(defaults.headers.is_empty()); assert!(defaults.resource_attributes.is_empty()); @@ -2098,6 +2104,192 @@ fn orphan_marks_become_zero_duration_spans() { ); } +#[test] +fn tool_projection_emits_generic_mark_as_parented_openinference_tool_span() { + let (provider, exporter) = make_provider(); + let mut processor = OpenInferenceEventProcessor::new_with_mark_projection( + provider.clone(), + "test-scope".to_string(), + MarkProjection::Tool, + ); + let parent_uuid = Uuid::now_v7(); + let start = make_start_event(parent_uuid, None, "agent-turn", ScopeType::Agent, None); + let mark = Event::Mark(MarkEvent::new( + BaseEvent::builder() + .parent_uuid(parent_uuid) + .name("plugin.output_compacted") + .data(json!({"count": 3})) + .build(), + Some(EventCategory::custom()), + Some( + CategoryProfile::builder() + .subtype("example.compaction") + .build(), + ), + )); + let end = make_end_event(parent_uuid, None, "agent-turn", ScopeType::Agent, None); + + processor.process(&start); + processor.process(&mark); + processor.process(&end); + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 2); + let parent = spans + .iter() + .find(|span| span.name.as_ref() == "agent-turn") + .unwrap(); + let projected = spans + .iter() + .find(|span| span.name.as_ref() == "mark:plugin.output_compacted") + .unwrap(); + assert!(parent.events.events.is_empty()); + assert_eq!(projected.parent_span_id, parent.span_context.span_id()); + assert_eq!(projected.start_time, projected.end_time); + + let attributes = attr_map(&projected.attributes); + assert_eq!( + attributes.get("openinference.span.kind"), + Some(&"TOOL".to_string()) + ); + assert_eq!( + attributes.get("nemo_relay.mark.projection"), + Some(&"tool".to_string()) + ); + assert_eq!( + attributes.get("nemo_relay.mark.data_json"), + Some(&"{\"count\":3}".to_string()) + ); + assert_eq!( + attributes.get("nemo_relay.mark.category"), + Some(&"custom".to_string()) + ); + assert_eq!( + attributes.get("nemo_relay.mark.category_profile_json"), + Some(&"{\"subtype\":\"example.compaction\"}".to_string()) + ); + assert!(!attributes.contains_key("nemo_relay.mark.orphan")); +} + +#[test] +fn tool_projection_exclusion_keeps_custom_mark_as_native_event() { + let (provider, exporter) = make_provider(); + let mut processor = OpenInferenceEventProcessor::new_with_mark_projection_and_exclusions( + provider.clone(), + "test-scope".to_string(), + MarkProjection::Tool, + vec!["plugin.excluded".to_string()], + ); + processor.process(&make_mark_event( + None, + "plugin.excluded", + Some(json!({"count": 3})), + )); + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 1); + let attributes = attr_map(&spans[0].attributes); + assert!(!attributes.contains_key("nemo_relay.mark.projection")); + assert_eq!( + attributes.get("openinference.span.kind"), + Some(&"CHAIN".to_string()) + ); +} + +#[test] +fn tool_projection_reuses_completed_parent_context_for_late_marks() { + let (provider, exporter) = make_provider(); + let mut processor = OpenInferenceEventProcessor::new_with_mark_projection( + provider.clone(), + "test-scope".to_string(), + MarkProjection::Tool, + ); + let parent_uuid = Uuid::now_v7(); + + processor.process(&make_start_event( + parent_uuid, + None, + "completed-tool", + ScopeType::Tool, + None, + )); + processor.process(&make_end_event( + parent_uuid, + None, + "completed-tool", + ScopeType::Tool, + None, + )); + processor.process(&make_mark_event( + Some(parent_uuid), + "plugin.late_checkpoint", + Some(json!({"status": "complete"})), + )); + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + let parent = spans + .iter() + .find(|span| span.name.as_ref() == "completed-tool") + .unwrap(); + let projected = spans + .iter() + .find(|span| span.name.as_ref() == "mark:plugin.late_checkpoint") + .unwrap(); + assert_eq!(projected.parent_span_id, parent.span_context.span_id()); + assert_eq!( + projected.span_context.trace_id(), + parent.span_context.trace_id() + ); + let attributes = attr_map(&projected.attributes); + assert_eq!( + attributes.get("nemo_relay.mark.orphan"), + Some(&"true".to_string()) + ); + assert_eq!( + attributes.get("openinference.span.kind"), + Some(&"TOOL".to_string()) + ); +} + +#[test] +fn tool_projection_keeps_llm_chunks_as_parent_span_events() { + let (provider, exporter) = make_provider(); + let mut processor = OpenInferenceEventProcessor::new_with_mark_projection( + provider.clone(), + "test-scope".to_string(), + MarkProjection::Tool, + ); + let parent_uuid = Uuid::now_v7(); + let start = make_start_event(parent_uuid, None, "llm", ScopeType::Llm, None); + let chunk = make_mark_event(Some(parent_uuid), "llm.chunk", Some(json!({"delta": "x"}))); + let aliased_chunk = Event::Mark(MarkEvent::new( + BaseEvent::builder() + .parent_uuid(parent_uuid) + .name("hook_mark") + .metadata(json!({"hook_event_name": "llm.chunk"})) + .data(json!({"delta": "y"})) + .build(), + None, + None, + )); + let end = make_end_event(parent_uuid, None, "llm", ScopeType::Llm, None); + + processor.process(&start); + processor.process(&chunk); + processor.process(&aliased_chunk); + processor.process(&end); + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 1); + assert_eq!(spans[0].events.events.len(), 2); + assert_eq!(spans[0].events.events[0].name.as_ref(), "llm.chunk"); + assert_eq!(spans[0].events.events[1].name.as_ref(), "hook_mark"); +} + #[test] fn late_parented_marks_reuse_completed_parent_trace_context() { let (provider, exporter) = make_provider(); @@ -2121,7 +2313,7 @@ fn late_parented_marks_reuse_completed_parent_trace_context() { )); processor.process(&make_mark_event( Some(tool_uuid), - "visor.tool_output_compressed", + "plugin.output_compacted", Some(json!({"estimated_tokens_saved": 42})), )); processor.force_flush().unwrap(); @@ -2134,7 +2326,7 @@ fn late_parented_marks_reuse_completed_parent_trace_context() { .unwrap(); let mark_span = spans .iter() - .find(|span| span.name.as_ref() == "mark:visor.tool_output_compressed") + .find(|span| span.name.as_ref() == "mark:plugin.output_compacted") .unwrap(); assert_eq!( diff --git a/crates/core/tests/unit/observability/otel_tests.rs b/crates/core/tests/unit/observability/otel_tests.rs index 31aee7be2..b03e3e7f5 100644 --- a/crates/core/tests/unit/observability/otel_tests.rs +++ b/crates/core/tests/unit/observability/otel_tests.rs @@ -374,6 +374,8 @@ fn config_defaults_and_builder_overrides_are_applied() { .with_service_namespace("agents") .with_service_version("1.2.3") .with_instrumentation_scope("demo-scope") + .with_mark_projection(MarkProjection::Tool) + .with_mark_exclude_names(["notification", "hook_mark"]) .with_timeout(Duration::from_millis(1250)); assert_eq!(config.transport, OtlpTransport::HttpBinary); @@ -393,12 +395,16 @@ fn config_defaults_and_builder_overrides_are_applied() { assert_eq!(config.service_namespace.as_deref(), Some("agents")); assert_eq!(config.service_version.as_deref(), Some("1.2.3")); assert_eq!(config.instrumentation_scope, "demo-scope"); + assert_eq!(config.mark_projection, MarkProjection::Tool); + assert_eq!(config.mark_exclude_names, vec!["notification", "hook_mark"]); assert_eq!(config.timeout, Duration::from_millis(1250)); let defaults = OpenTelemetryConfig::default(); assert_eq!(defaults.transport, OtlpTransport::HttpBinary); assert_eq!(defaults.service_name, "nemo-relay"); assert_eq!(defaults.instrumentation_scope, "nemo-relay-otel"); + assert_eq!(defaults.mark_projection, MarkProjection::Inherit); + assert_eq!(defaults.mark_exclude_names, vec!["llm.chunk"]); assert_eq!(defaults.timeout, Duration::from_secs(3)); assert!(defaults.headers.is_empty()); assert!(defaults.resource_attributes.is_empty()); @@ -776,6 +782,189 @@ fn orphan_marks_become_zero_duration_spans() { ); } +#[test] +fn tool_projection_emits_generic_mark_as_parented_zero_duration_span() { + let (provider, exporter) = make_provider(); + let mut processor = OtelEventProcessor::new_with_mark_projection( + provider.clone(), + "test-scope".to_string(), + MarkProjection::Tool, + ); + let parent_uuid = Uuid::now_v7(); + let start = make_start_event(parent_uuid, None, "agent-turn", ScopeType::Agent, None); + let mark = Event::Mark(MarkEvent::new( + BaseEvent::builder() + .parent_uuid(parent_uuid) + .name("plugin.output_compacted") + .data(json!({"count": 3})) + .build(), + Some(EventCategory::custom()), + Some( + CategoryProfile::builder() + .subtype("example.compaction") + .build(), + ), + )); + let end = make_end_event(parent_uuid, None, "agent-turn", ScopeType::Agent, None); + + processor.process(&start); + processor.process(&mark); + processor.process(&end); + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 2); + let parent = spans + .iter() + .find(|span| span.name.as_ref() == "agent-turn") + .unwrap(); + let projected = spans + .iter() + .find(|span| span.name.as_ref() == "mark:plugin.output_compacted") + .unwrap(); + assert!(parent.events.events.is_empty()); + assert_eq!(projected.parent_span_id, parent.span_context.span_id()); + assert_eq!(projected.start_time, projected.end_time); + assert!(parent.start_time <= projected.start_time); + assert!(projected.end_time <= parent.end_time); + + let attributes = attr_map(&projected.attributes); + assert_eq!( + attributes.get("nemo_relay.mark.projection"), + Some(&"tool".to_string()) + ); + assert_eq!( + attributes.get("nemo_relay.scope_type"), + Some(&"tool".to_string()) + ); + assert_eq!( + attributes.get("nemo_relay.mark.data_json"), + Some(&"{\"count\":3}".to_string()) + ); + assert_eq!( + attributes.get("nemo_relay.mark.category"), + Some(&"custom".to_string()) + ); + assert_eq!( + attributes.get("nemo_relay.mark.category_profile_json"), + Some(&"{\"subtype\":\"example.compaction\"}".to_string()) + ); + assert!(!attributes.contains_key("nemo_relay.mark.orphan")); +} + +#[test] +fn tool_projection_exclusion_keeps_custom_mark_as_native_event() { + let (provider, exporter) = make_provider(); + let mut processor = OtelEventProcessor::new_with_mark_projection_and_exclusions( + provider.clone(), + "test-scope".to_string(), + MarkProjection::Tool, + vec!["plugin.excluded".to_string()], + ); + processor.process(&make_mark_event( + None, + "plugin.excluded", + Some(json!({"count": 3})), + )); + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 1); + let attributes = attr_map(&spans[0].attributes); + assert!(!attributes.contains_key("nemo_relay.mark.projection")); + assert_eq!( + attributes.get("nemo_relay.mark.orphan"), + Some(&"true".to_string()) + ); +} + +#[test] +fn tool_projection_reuses_completed_parent_context_for_late_marks() { + let (provider, exporter) = make_provider(); + let mut processor = OtelEventProcessor::new_with_mark_projection( + provider.clone(), + "test-scope".to_string(), + MarkProjection::Tool, + ); + let parent_uuid = Uuid::now_v7(); + + processor.process(&make_start_event( + parent_uuid, + None, + "completed-tool", + ScopeType::Tool, + None, + )); + processor.process(&make_end_event( + parent_uuid, + None, + "completed-tool", + ScopeType::Tool, + None, + )); + processor.process(&make_mark_event( + Some(parent_uuid), + "plugin.late_checkpoint", + Some(json!({"status": "complete"})), + )); + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + let parent = spans + .iter() + .find(|span| span.name.as_ref() == "completed-tool") + .unwrap(); + let projected = spans + .iter() + .find(|span| span.name.as_ref() == "mark:plugin.late_checkpoint") + .unwrap(); + assert_eq!(projected.parent_span_id, parent.span_context.span_id()); + assert_eq!( + projected.span_context.trace_id(), + parent.span_context.trace_id() + ); + assert_eq!( + attr_map(&projected.attributes).get("nemo_relay.mark.orphan"), + Some(&"true".to_string()) + ); +} + +#[test] +fn tool_projection_keeps_llm_chunks_as_parent_span_events() { + let (provider, exporter) = make_provider(); + let mut processor = OtelEventProcessor::new_with_mark_projection( + provider.clone(), + "test-scope".to_string(), + MarkProjection::Tool, + ); + let parent_uuid = Uuid::now_v7(); + let start = make_start_event(parent_uuid, None, "llm", ScopeType::Llm, None); + let chunk = make_mark_event(Some(parent_uuid), "llm.chunk", Some(json!({"delta": "x"}))); + let aliased_chunk = Event::Mark(MarkEvent::new( + BaseEvent::builder() + .parent_uuid(parent_uuid) + .name("hook_mark") + .metadata(json!({"hook_event_name": "llm.chunk"})) + .data(json!({"delta": "y"})) + .build(), + None, + None, + )); + let end = make_end_event(parent_uuid, None, "llm", ScopeType::Llm, None); + + processor.process(&start); + processor.process(&chunk); + processor.process(&aliased_chunk); + processor.process(&end); + processor.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + assert_eq!(spans.len(), 1); + assert_eq!(spans[0].events.events.len(), 2); + assert_eq!(spans[0].events.events[0].name.as_ref(), "llm.chunk"); + assert_eq!(spans[0].events.events[1].name.as_ref(), "hook_mark"); +} + #[test] fn late_parented_marks_reuse_completed_parent_trace_context() { let (provider, exporter) = make_provider(); @@ -798,7 +987,7 @@ fn late_parented_marks_reuse_completed_parent_trace_context() { )); processor.process(&make_mark_event( Some(tool_uuid), - "visor.tool_output_compressed", + "plugin.output_compacted", Some(json!({"estimated_tokens_saved": 42})), )); processor.force_flush().unwrap(); @@ -811,7 +1000,7 @@ fn late_parented_marks_reuse_completed_parent_trace_context() { .unwrap(); let mark_span = spans .iter() - .find(|span| span.name.as_ref() == "mark:visor.tool_output_compressed") + .find(|span| span.name.as_ref() == "mark:plugin.output_compacted") .unwrap(); assert_eq!( diff --git a/crates/core/tests/unit/observability/plugin_component_tests.rs b/crates/core/tests/unit/observability/plugin_component_tests.rs index 7b02d1588..c43d31623 100644 --- a/crates/core/tests/unit/observability/plugin_component_tests.rs +++ b/crates/core/tests/unit/observability/plugin_component_tests.rs @@ -122,21 +122,48 @@ fn schema_has_property(schema: &Json, name: &str) -> bool { #[cfg(feature = "schema")] fn schema_property_has_enum(schema: &Json, name: &str, expected: &[&str]) -> bool { - schema_property(schema, name) - .and_then(|property| property.get("enum")) - .and_then(Json::as_array) - .is_some_and(|values| { - expected - .iter() - .all(|expected| values.iter().any(|value| value == *expected)) - }) + schema_property_matches(schema, name, &|property| { + property + .get("enum") + .and_then(Json::as_array) + .is_some_and(|values| { + expected + .iter() + .all(|expected| values.iter().any(|value| value == *expected)) + }) + }) } #[cfg(feature = "schema")] fn schema_property_has_default(schema: &Json, name: &str, expected: Json) -> bool { - schema_property(schema, name) - .and_then(|property| property.get("default")) - .is_some_and(|default| default == &expected) + schema_property_matches(schema, name, &|property| { + property + .get("default") + .is_some_and(|default| default == &expected) + }) +} + +#[cfg(feature = "schema")] +fn schema_property_matches(schema: &Json, name: &str, predicate: &impl Fn(&Json) -> bool) -> bool { + match schema { + Json::Object(object) => { + if object + .get("properties") + .and_then(Json::as_object) + .and_then(|properties| properties.get(name)) + .is_some_and(predicate) + { + return true; + } + object + .values() + .any(|value| schema_property_matches(value, name, predicate)) + } + Json::Array(values) => values + .iter() + .any(|value| schema_property_matches(value, name, predicate)), + _ => false, + } } #[cfg(feature = "schema")] @@ -193,6 +220,8 @@ fn default_config_and_component_conversion_cover_public_shape() { let otlp = OtlpSectionConfig::default(); assert!(!otlp.enabled); + assert_eq!(otlp.mark_projection, MarkProjection::Inherit); + assert_eq!(otlp.mark_exclude_names, vec!["llm.chunk"]); assert_eq!(otlp.transport, "http_binary"); assert_eq!(otlp.service_name, "nemo-relay"); assert_eq!(otlp.timeout_millis, 3_000); @@ -211,6 +240,55 @@ fn default_config_and_component_conversion_cover_public_shape() { assert_eq!(generic.config["atif"]["agent_name"], json!("NeMo Relay")); } +#[test] +fn mark_projection_parses_for_otlp_and_rejects_unknown_values() { + let otlp: OtlpSectionConfig = serde_json::from_value(json!({ + "mark_projection": "tool" + })) + .unwrap(); + assert_eq!(otlp.mark_projection, MarkProjection::Tool); + + let event: OtlpSectionConfig = serde_json::from_value(json!({ + "mark_projection": "event", + "mark_exclude_names": [] + })) + .unwrap(); + assert_eq!(event.mark_projection, MarkProjection::Event); + assert!(event.mark_exclude_names.is_empty()); + + let custom_exclusions: OtlpSectionConfig = serde_json::from_value(json!({ + "mark_projection": "tool", + "mark_exclude_names": ["notification", "hook_mark"] + })) + .unwrap(); + assert_eq!( + custom_exclusions.mark_exclude_names, + vec!["notification", "hook_mark"] + ); + + let error = serde_json::from_value::(json!({ + "mark_projection": "span" + })) + .unwrap_err(); + assert!(error.to_string().contains("unknown variant `span`")); + + let report = validate_plugin_config(&plugin_config(json!({ + "opentelemetry": {"mark_projection": "span"} + }))); + assert!(report.has_errors()); + assert!(report.diagnostics.iter().any(|diagnostic| diagnostic.code + == "observability.invalid_plugin_config" + && diagnostic.message.contains("unknown variant `span`"))); + + let report = validate_plugin_config(&plugin_config(json!({ + "atif": {"mark_projection": "tool"} + }))); + assert!(report.diagnostics.iter().any(|diagnostic| { + diagnostic.code == "observability.unknown_field" + && diagnostic.field.as_deref() == Some("mark_projection") + })); +} + #[cfg(feature = "schema")] #[test] fn schema_contains_every_supported_observability_option() { @@ -230,6 +308,8 @@ fn schema_contains_every_supported_observability_option() { "agent_name", "agent_version", "model_name", + "mark_projection", + "mark_exclude_names", "tool_definitions", "extra", "filename_template", @@ -262,6 +342,11 @@ fn schema_contains_every_supported_observability_option() { "transport", &["http_binary", "grpc"] )); + assert!(schema_property_has_enum( + &schema, + "mark_projection", + &["inherit", "event", "tool"] + )); assert!(schema_property_has_default( &schema, "mode", @@ -894,7 +979,7 @@ fn atif_routes_global_descendant_events_by_parent_uuid() { } #[test] -fn atif_keeps_openclaw_child_only_fallback_as_a_top_level_trajectory() { +fn atif_writes_openclaw_child_only_fallback_without_mark_steps() { let _guard = crate::observability::test_mutex().lock().unwrap(); reset_runtime(); let dir = temp_dir("observability-atif-openclaw-child-fallback"); @@ -998,8 +1083,7 @@ fn atif_keeps_openclaw_child_only_fallback_as_a_top_level_trajectory() { let value: Json = serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap(); assert_eq!(value["trajectory_id"], child_uuid.to_string()); - assert_eq!(value["steps"].as_array().unwrap().len(), 1); - assert_eq!(value["steps"][0]["message"], "worker-started"); + assert!(value["steps"].as_array().unwrap().is_empty()); assert!( value.get("subagent_trajectories").is_none() || value["subagent_trajectories"].is_null() ); diff --git a/crates/node/observability.d.ts b/crates/node/observability.d.ts index f12ede6ec..eb15be134 100644 --- a/crates/node/observability.d.ts +++ b/crates/node/observability.d.ts @@ -56,6 +56,8 @@ export interface AtifConfig { export interface OtlpConfig { enabled?: boolean; + mark_projection?: 'inherit' | 'event' | 'tool'; + mark_exclude_names?: string[]; transport?: 'http_binary' | 'grpc' | string; endpoint?: string; headers?: Record; diff --git a/crates/node/observability.js b/crates/node/observability.js index 7ce69c935..97e5c8415 100644 --- a/crates/node/observability.js +++ b/crates/node/observability.js @@ -57,6 +57,8 @@ function atifConfig(config = {}) { function otlpConfig(config = {}) { return { enabled: false, + mark_projection: 'inherit', + mark_exclude_names: ['llm.chunk'], transport: 'http_binary', headers: {}, resource_attributes: {}, diff --git a/crates/node/tests/observability_plugin_tests.mjs b/crates/node/tests/observability_plugin_tests.mjs index e0741d079..94b825031 100644 --- a/crates/node/tests/observability_plugin_tests.mjs +++ b/crates/node/tests/observability_plugin_tests.mjs @@ -29,12 +29,15 @@ describe('observability plugin helpers', () => { }); assert.deepEqual(observability.otlpConfig(), { enabled: false, + mark_projection: 'inherit', + mark_exclude_names: ['llm.chunk'], transport: 'http_binary', headers: {}, resource_attributes: {}, service_name: 'nemo-relay', timeout_millis: 3000, }); + assert.equal(observability.otlpConfig({ mark_projection: 'tool' }).mark_projection, 'tool'); const component = observability.ComponentSpec({ version: 1, atof: observability.atofConfig() }); assert.equal(component.kind, observability.OBSERVABILITY_PLUGIN_KIND); diff --git a/docs/configure-plugins/observability/atif.mdx b/docs/configure-plugins/observability/atif.mdx index c3e16c9ff..d56617c69 100644 --- a/docs/configure-plugins/observability/atif.mdx +++ b/docs/configure-plugins/observability/atif.mdx @@ -43,6 +43,11 @@ This configuration writes a trajectory file such as `logs/trajectory-.json` for each top-level Agent scope or supported coding-agent turn scope. +ATIF contains agent interaction steps, tool calls, and observations. Relay mark +events are point-in-time telemetry rather than trajectory steps, so the ATIF +exporter omits them. Use ATOF to retain the canonical mark event stream, or an +OTLP exporter when marks must remain correlated with trace data. + ## Fields The following table describes the top-level ATIF settings: diff --git a/docs/configure-plugins/observability/openinference.mdx b/docs/configure-plugins/observability/openinference.mdx index 56f3e8edc..423d5435f 100644 --- a/docs/configure-plugins/observability/openinference.mdx +++ b/docs/configure-plugins/observability/openinference.mdx @@ -61,6 +61,8 @@ OpenInference uses the same OTLP section shape as | Field | Default | Notes | |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | +| `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces span events; `tool` emits zero-duration OpenInference `TOOL` spans, parented as children when context is available. | +| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain exporter-native handling. Metadata `hook_event_name` aliases are also matched. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | | `headers` | `{}` | String-to-string exporter headers. | @@ -83,6 +85,21 @@ and invocation parameters are emitted when typed codec annotations supply them. Exported LLM attributes exclude request headers and other non-observable transport metadata. +The default `inherit` projection preserves exporter-native mark handling: a +mark with an active parent span is a span event, while an orphan mark is a +standalone zero-duration mark span. `mark_projection = "event"` explicitly +selects that representation. With `mark_projection = "tool"`, each eligible +mark becomes a zero-duration span (a child span when parent context is +available) with `openinference.span.kind = "TOOL"` and the original mark +payload, category, profile, timestamp, and parent identifiers retained as +attributes. High-volume `llm.chunk` marks retain exporter-native handling by +default and when excluded from tool projection. + +Add other event names to `mark_exclude_names` for a backend to keep those +marks in exporter-native form rather than visible tool children. The exclusion +list affects only tool projection; it does not remove mark payload or metadata. +Set `mark_exclude_names = []` to disable the default `llm.chunk` exclusion. + Each lifecycle span includes `nemo_relay.uuid` and `nemo_relay.parent_uuid` attributes. These values match ATIF `step.extra.ancestry.function_id` and `step.extra.ancestry.parent_id` for the same events. For plugin-managed ATIF, diff --git a/docs/configure-plugins/observability/opentelemetry.mdx b/docs/configure-plugins/observability/opentelemetry.mdx index 70606a24d..126c689cd 100644 --- a/docs/configure-plugins/observability/opentelemetry.mdx +++ b/docs/configure-plugins/observability/opentelemetry.mdx @@ -55,6 +55,8 @@ The following table describes OpenTelemetry exporter settings: | Field | Default | Notes | |---|---|---| | `enabled` | `false` | Must be `true` to construct and register the subscriber. | +| `mark_projection` | `inherit` | `inherit` uses exporter-native handling; `event` forces span events; `tool` emits zero-duration spans, parented as children when context is available, for trace-tree visibility. | +| `mark_exclude_names` | `["llm.chunk"]` | Mark names excluded from `tool` projection; excluded marks retain exporter-native handling. Metadata `hook_event_name` aliases are also matched. | | `transport` | `http_binary` | `http_binary` or `grpc`. | | `endpoint` | Exporter default | OTLP endpoint. | | `headers` | `{}` | String-to-string exporter headers. | @@ -71,6 +73,21 @@ The collector should receive OTLP trace export requests. The tracing backend should show spans for NeMo Relay scopes, tools, LLM calls, and marks grouped by root scope. +The default `inherit` projection follows exporter-native handling: a mark with +an active parent span is a span event, while an orphan mark is a standalone +zero-duration `mark:` span. `mark_projection = "event"` explicitly selects +that representation. Set `mark_projection = "tool"` only when the target +backend needs eligible marks represented as visible child spans. Projected +marks use `SpanKind::Internal`, carry +`nemo_relay.mark.projection = "tool"`, and retain mark UUID, parentage, +category/profile, and payload attributes. High-volume `llm.chunk` marks retain +exporter-native handling by default and when excluded from tool projection. + +Add other event names to `mark_exclude_names` for a backend to keep those +marks in exporter-native form rather than visible tool children. The exclusion +list affects only tool projection; it does not remove mark payload or metadata. +Set `mark_exclude_names = []` to disable the default `llm.chunk` exclusion. + Each lifecycle span includes `nemo_relay.uuid` and `nemo_relay.parent_uuid` attributes. These values match ATIF `step.extra.ancestry.function_id` and `step.extra.ancestry.parent_id` for the same events. For plugin-managed ATIF, diff --git a/docs/resources/glossary.mdx b/docs/resources/glossary.mdx index 72aa80ed3..c91106710 100644 --- a/docs/resources/glossary.mdx +++ b/docs/resources/glossary.mdx @@ -45,7 +45,7 @@ An **adaptive state backend** stores observations and learned state for adaptive **Agent Trajectory Observability Format (ATOF)** -**Agent Trajectory Observability Format (ATOF)** is the canonical event format NeMo Relay emits for scope lifecycle events and mark events. Subscribers and exporters consume ATOF events before translating them into downstream observability formats such as ATIF trajectories, OpenTelemetry traces, or OpenInference spans. +**Agent Trajectory Observability Format (ATOF)** is the canonical event format NeMo Relay emits for scope lifecycle events and mark events. Subscribers and exporters consume ATOF events before translating compatible scope data into ATIF trajectories and trace data into OpenTelemetry or OpenInference. **Annotated Request And Response Data** diff --git a/go/nemo_relay/observability_plugin.go b/go/nemo_relay/observability_plugin.go index 3660dfe4b..c79488919 100644 --- a/go/nemo_relay/observability_plugin.go +++ b/go/nemo_relay/observability_plugin.go @@ -8,6 +8,18 @@ import "encoding/json" // ObservabilityPluginKind is the top-level plugin kind used by the core observability component. const ObservabilityPluginKind = "observability" +// ObservabilityMarkProjection controls how point-in-time marks are exported. +type ObservabilityMarkProjection string + +const ( + // ObservabilityMarkProjectionInherit preserves exporter-native mark handling. + ObservabilityMarkProjectionInherit ObservabilityMarkProjection = "inherit" + // ObservabilityMarkProjectionEvent preserves marks as events. + ObservabilityMarkProjectionEvent ObservabilityMarkProjection = "event" + // ObservabilityMarkProjectionTool emits visible tool projections. + ObservabilityMarkProjectionTool ObservabilityMarkProjection = "tool" +) + // ObservabilityConfig is the canonical Go shape for the observability plugin config document. type ObservabilityConfig struct { Version uint32 `json:"version,omitempty"` @@ -128,16 +140,42 @@ func (config ObservabilityHttpStorageConfig) MarshalJSON() ([]byte, error) { // ObservabilityOtlpConfig configures OpenTelemetry or OpenInference OTLP export. type ObservabilityOtlpConfig struct { - Enabled bool `json:"enabled,omitempty"` - Transport string `json:"transport,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - Headers map[string]string `json:"headers,omitempty"` - ResourceAttributes map[string]string `json:"resource_attributes,omitempty"` - ServiceName string `json:"service_name,omitempty"` - ServiceNamespace string `json:"service_namespace,omitempty"` - ServiceVersion string `json:"service_version,omitempty"` - InstrumentationScope string `json:"instrumentation_scope,omitempty"` - TimeoutMillis uint64 `json:"timeout_millis,omitempty"` + Enabled bool `json:"enabled,omitempty"` + MarkProjection ObservabilityMarkProjection `json:"mark_projection,omitempty"` + MarkExcludeNames []string `json:"mark_exclude_names,omitempty"` + Transport string `json:"transport,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + ResourceAttributes map[string]string `json:"resource_attributes,omitempty"` + ServiceName string `json:"service_name,omitempty"` + ServiceNamespace string `json:"service_namespace,omitempty"` + ServiceVersion string `json:"service_version,omitempty"` + InstrumentationScope string `json:"instrumentation_scope,omitempty"` + TimeoutMillis uint64 `json:"timeout_millis,omitempty"` +} + +// MarshalJSON preserves the distinction between a nil exclusion list, which +// inherits the core default, and an explicitly empty list, which disables all +// default exclusions. +func (config ObservabilityOtlpConfig) MarshalJSON() ([]byte, error) { + type alias ObservabilityOtlpConfig + payload, err := json.Marshal(alias(config)) + if err != nil { + return nil, err + } + + var object map[string]json.RawMessage + if err := json.Unmarshal(payload, &object); err != nil { + return nil, err + } + if config.MarkExcludeNames != nil { + exclusions, err := json.Marshal(config.MarkExcludeNames) + if err != nil { + return nil, err + } + object["mark_exclude_names"] = exclusions + } + return json.Marshal(object) } // ObservabilityComponentSpec wraps one observability config as a top-level plugin component. @@ -181,6 +219,8 @@ func NewObservabilityHttpStorageConfig(endpoint string) ObservabilityHttpStorage func NewObservabilityOtlpConfig() ObservabilityOtlpConfig { return ObservabilityOtlpConfig{ Transport: "http_binary", + MarkProjection: ObservabilityMarkProjectionInherit, + MarkExcludeNames: []string{"llm.chunk"}, Headers: map[string]string{}, ResourceAttributes: map[string]string{}, ServiceName: "nemo-relay", diff --git a/go/nemo_relay/observability_plugin_test.go b/go/nemo_relay/observability_plugin_test.go index 0ad8e306f..7c94c33b9 100644 --- a/go/nemo_relay/observability_plugin_test.go +++ b/go/nemo_relay/observability_plugin_test.go @@ -64,12 +64,14 @@ func TestObservabilityConfigHelpers(t *testing.T) { httpStorage, } otlp := NewObservabilityOtlpConfig() - if otlp.Enabled || otlp.Transport != "http_binary" || otlp.ServiceName != "nemo-relay" || otlp.TimeoutMillis != 3000 { + if otlp.Enabled || otlp.MarkProjection != ObservabilityMarkProjectionInherit || len(otlp.MarkExcludeNames) != 1 || otlp.MarkExcludeNames[0] != "llm.chunk" || otlp.Transport != "http_binary" || otlp.ServiceName != "nemo-relay" || otlp.TimeoutMillis != 3000 { t.Fatalf("unexpected OTLP defaults: %#v", otlp) } + otlp.MarkProjection = ObservabilityMarkProjectionTool config.Atof = &atof config.Atif = &atif + config.OpenTelemetry = &otlp wrapped := ObservabilityComponent(config) if wrapped.Kind != ObservabilityPluginKind || !wrapped.Enabled { t.Fatalf("unexpected component wrapper: %#v", wrapped) @@ -94,6 +96,29 @@ func TestObservabilityConfigHelpers(t *testing.T) { t.Fatalf("expected field_name_policy in serialized component, got %s", serialized) } assertWrappedAtifStorageConfig(t, wrapped.Config["atif"].(map[string]any)) + if wrapped.Config["opentelemetry"].(map[string]any)["mark_projection"] != "tool" { + t.Fatalf("expected tool mark projection in serialized config: %#v", wrapped.Config) + } +} + +func TestObservabilityOtlpConfigPreservesExplicitEmptyMarkExclusions(t *testing.T) { + inherited, err := json.Marshal(ObservabilityOtlpConfig{}) + if err != nil { + t.Fatalf("marshal zero-value OTLP config failed: %v", err) + } + if strings.Contains(string(inherited), `"mark_exclude_names"`) { + t.Fatalf("nil exclusions should inherit the core default: %s", inherited) + } + + config := NewObservabilityOtlpConfig() + config.MarkExcludeNames = []string{} + explicitEmpty, err := json.Marshal(config) + if err != nil { + t.Fatalf("marshal OTLP config with empty exclusions failed: %v", err) + } + if !strings.Contains(string(explicitEmpty), `"mark_exclude_names":[]`) { + t.Fatalf("explicit empty exclusions should be preserved: %s", explicitEmpty) + } } func assertS3StorageConfig(t *testing.T, storage ObservabilityS3StorageConfig) { diff --git a/python/nemo_relay/observability.py b/python/nemo_relay/observability.py index 38d90e9c0..ad837a0a1 100644 --- a/python/nemo_relay/observability.py +++ b/python/nemo_relay/observability.py @@ -10,6 +10,8 @@ from nemo_relay import Json, JsonObject, UnsupportedBehavior +MarkProjection = Literal["inherit", "event", "tool"] + class _SupportsToDict(Protocol): def to_dict(self) -> JsonObject: ... @@ -194,6 +196,8 @@ class OtlpConfig: """Shared OpenTelemetry/OpenInference OTLP export settings.""" enabled: bool = False + mark_projection: MarkProjection = "inherit" + mark_exclude_names: list[str] = field(default_factory=lambda: ["llm.chunk"]) transport: Literal["http_binary", "grpc"] = "http_binary" endpoint: str | None = None headers: dict[str, str] = field(default_factory=dict) @@ -209,6 +213,8 @@ def to_dict(self) -> JsonObject: return _normalize_object( { "enabled": self.enabled, + "mark_projection": self.mark_projection, + "mark_exclude_names": self.mark_exclude_names, "transport": self.transport, "endpoint": self.endpoint, "headers": self.headers, @@ -272,6 +278,7 @@ def to_dict(self) -> JsonObject: "AtofConfig", "AtifConfig", "HttpStorageConfig", + "MarkProjection", "S3StorageConfig", "OtlpConfig", "ObservabilityConfig", diff --git a/python/nemo_relay/observability.pyi b/python/nemo_relay/observability.pyi index 11e5b4d60..f562a10a1 100644 --- a/python/nemo_relay/observability.pyi +++ b/python/nemo_relay/observability.pyi @@ -10,6 +10,8 @@ from typing import Literal from nemo_relay import JsonObject, UnsupportedBehavior +MarkProjection = Literal["inherit", "event", "tool"] + @dataclass(slots=True) class ConfigPolicy: unknown_component: UnsupportedBehavior = ... @@ -71,6 +73,8 @@ class AtifConfig: @dataclass(slots=True) class OtlpConfig: enabled: bool = ... + mark_projection: MarkProjection = ... + mark_exclude_names: list[str] = ... transport: Literal["http_binary", "grpc"] = ... endpoint: str | None = ... headers: dict[str, str] = field(default_factory=dict) diff --git a/python/tests/test_observability_plugin.py b/python/tests/test_observability_plugin.py index 7dd34841b..e536fcaf8 100644 --- a/python/tests/test_observability_plugin.py +++ b/python/tests/test_observability_plugin.py @@ -38,12 +38,15 @@ def test_defaults_and_component_wrapper(self): } assert OtlpConfig().to_dict() == { "enabled": False, + "mark_projection": "inherit", + "mark_exclude_names": ["llm.chunk"], "transport": "http_binary", "headers": {}, "resource_attributes": {}, "service_name": "nemo-relay", "timeout_millis": 3000, } + assert OtlpConfig(mark_projection="tool").to_dict()["mark_projection"] == "tool" wrapped = ComponentSpec(ObservabilityConfig(atof=AtofConfig())).to_dict() assert wrapped["kind"] == OBSERVABILITY_PLUGIN_KIND