Telemetry foundations & Tracing utilities setup

.gitignore

@@ -189,4 +189,8 @@ docs
 # vscode workspace config
 agents-js.code-workspace
 
-examples/src/test_*.ts
+examples/src/test_*.ts
+
+# Ignore all markdown files except root README
+*.md
+!README.md

agents/package.json

@@ -50,6 +50,12 @@
     "@livekit/mutex": "^1.1.1",
     "@livekit/protocol": "^1.41.0",
     "@livekit/typed-emitter": "^3.0.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.54.0",
+    "@opentelemetry/resources": "^1.28.0",
+    "@opentelemetry/sdk-trace-base": "^1.28.0",
+    "@opentelemetry/sdk-trace-node": "^1.28.0",
+    "@opentelemetry/semantic-conventions": "^1.28.0",
     "@types/pidusage": "^2.0.5",
     "commander": "^12.0.0",
     "fluent-ffmpeg": "^2.1.3",

agents/src/index.ts

@@ -16,6 +16,7 @@ import * as llm from './llm/index.js';
 import * as metrics from './metrics/index.js';
 import * as stream from './stream/index.js';
 import * as stt from './stt/index.js';
+import * as telemetry from './telemetry/index.js';
 import * as tokenize from './tokenize/index.js';
 import * as tts from './tts/index.js';
 import * as voice from './voice/index.js';
@@ -34,4 +35,4 @@ export * from './vad.js';
 export * from './version.js';
 export * from './worker.js';
 
-export { cli, inference, ipc, llm, metrics, stream, stt, tokenize, tts, voice };
+export { cli, inference, ipc, llm, metrics, stream, stt, telemetry, tokenize, tts, voice };

agents/src/job.ts

@@ -81,6 +81,8 @@ export class FunctionExistsError extends Error {
 }
 
 /** The job and environment context as seen by the agent, accessible by the entrypoint function. */
+// TODO(brian): PR3 - Add @tracer.startActiveSpan('job_entrypoint') wrapper in entrypoint
+// TODO(brian): PR5 - Add uploadSessionReport() call in cleanup/session end
 export class JobContext {
   #proc: JobProcess;
   #info: RunningJobInfo;
@@ -245,6 +247,7 @@ export class JobContext {
     }
 
     // TODO(brian): implement and check recorder io
+    // TODO(brian): PR5 - Ensure chat history serialization includes all required fields (use sessionReportToJSON helper)
 
     return createSessionReport({
       jobId: this.job.id,
@@ -267,7 +270,8 @@ export class JobContext {
 
     // TODO(brian): Implement CLI/console
 
-    // TODO(brian): Implement session report upload to LiveKit Cloud
+    // TODO(brian): PR5 - Call uploadSessionReport() if report.enableUserDataTraining is true
+    // TODO(brian): PR5 - Upload includes: multipart form with header (protobuf), chat_history (JSON), and audio recording (if available)
 
     this.#logger.debug('Session ended, report generated', {
       jobId: report.jobId,

agents/src/llm/llm.ts

@@ -136,8 +136,10 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
   }
 
   private async mainTask() {
+    // TODO(brian): PR3 - Add span wrapping: tracer.startActiveSpan('llm_request', ..., { endOnExit: false })
     for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
       try {
+        // TODO(brian): PR3 - Add span for retry attempts: tracer.startActiveSpan('llm_request_run', ...)
         return await this.run();
       } catch (error) {
         if (error instanceof APIError) {

agents/src/log.ts

@@ -42,4 +42,5 @@ export const initializeLogger = ({ pretty, level }: LoggerOptions) => {
   if (level) {
     logger.level = level;
   }
+  // TODO(brian): PR4 - Add Pino bridge to OTEL LoggingHandler for structured logging integration
 };

agents/src/telemetry/index.ts

@@ -0,0 +1,11 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// TODO(brian): PR2 - Add setupCloudTracer, MetadataSpanProcessor exports
+// TODO(brian): PR4 - Add logging integration exports
+// TODO(brian): PR5 - Add uploadSessionReport export
+
+export * as traceTypes from './trace_types.js';
+export { setTracerProvider, tracer, type SpanStartOptions as StartSpanOptions } from './traces.js';
+export { recordException, recordRealtimeMetrics } from './utils.js';

agents/src/telemetry/trace_types.ts

@@ -0,0 +1,88 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// LiveKit custom attributes
+export const ATTR_SPEECH_ID = 'lk.speech_id';
+export const ATTR_AGENT_LABEL = 'lk.agent_label';
+export const ATTR_START_TIME = 'lk.start_time';
+export const ATTR_END_TIME = 'lk.end_time';
+export const ATTR_RETRY_COUNT = 'lk.retry_count';
+
+export const ATTR_PARTICIPANT_ID = 'lk.participant_id';
+export const ATTR_PARTICIPANT_IDENTITY = 'lk.participant_identity';
+export const ATTR_PARTICIPANT_KIND = 'lk.participant_kind';
+
+// session start
+export const ATTR_JOB_ID = 'lk.job_id';
+export const ATTR_AGENT_NAME = 'lk.agent_name';
+export const ATTR_ROOM_NAME = 'lk.room_name';
+export const ATTR_SESSION_OPTIONS = 'lk.session_options';
+
+// assistant turn
+export const ATTR_USER_INPUT = 'lk.user_input';
+export const ATTR_INSTRUCTIONS = 'lk.instructions';
+export const ATTR_SPEECH_INTERRUPTED = 'lk.interrupted';
+
+// llm node
+export const ATTR_CHAT_CTX = 'lk.chat_ctx';
+export const ATTR_FUNCTION_TOOLS = 'lk.function_tools';
+export const ATTR_RESPONSE_TEXT = 'lk.response.text';
+export const ATTR_RESPONSE_FUNCTION_CALLS = 'lk.response.function_calls';
+
+// function tool
+export const ATTR_FUNCTION_TOOL_NAME = 'lk.function_tool.name';
+export const ATTR_FUNCTION_TOOL_ARGS = 'lk.function_tool.arguments';
+export const ATTR_FUNCTION_TOOL_IS_ERROR = 'lk.function_tool.is_error';
+export const ATTR_FUNCTION_TOOL_OUTPUT = 'lk.function_tool.output';
+
+// tts node
+export const ATTR_TTS_INPUT_TEXT = 'lk.input_text';
+export const ATTR_TTS_STREAMING = 'lk.tts.streaming';
+export const ATTR_TTS_LABEL = 'lk.tts.label';
+
+// eou detection
+export const ATTR_EOU_PROBABILITY = 'lk.eou.probability';
+export const ATTR_EOU_UNLIKELY_THRESHOLD = 'lk.eou.unlikely_threshold';
+export const ATTR_EOU_DELAY = 'lk.eou.endpointing_delay';
+export const ATTR_EOU_LANGUAGE = 'lk.eou.language';
+export const ATTR_USER_TRANSCRIPT = 'lk.user_transcript';
+export const ATTR_TRANSCRIPT_CONFIDENCE = 'lk.transcript_confidence';
+export const ATTR_TRANSCRIPTION_DELAY = 'lk.transcription_delay';
+export const ATTR_END_OF_TURN_DELAY = 'lk.end_of_turn_delay';
+
+// metrics
+export const ATTR_LLM_METRICS = 'lk.llm_metrics';
+export const ATTR_TTS_METRICS = 'lk.tts_metrics';
+export const ATTR_REALTIME_MODEL_METRICS = 'lk.realtime_model_metrics';
+
+// OpenTelemetry GenAI attributes
+// OpenTelemetry specification: https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/
+export const ATTR_GEN_AI_OPERATION_NAME = 'gen_ai.operation.name';
+export const ATTR_GEN_AI_REQUEST_MODEL = 'gen_ai.request.model';
+export const ATTR_GEN_AI_USAGE_INPUT_TOKENS = 'gen_ai.usage.input_tokens';
+export const ATTR_GEN_AI_USAGE_OUTPUT_TOKENS = 'gen_ai.usage.output_tokens';
+
+// Unofficial OpenTelemetry GenAI attributes, recognized by LangFuse
+// https://langfuse.com/integrations/native/opentelemetry#usage
+// but not yet in the official OpenTelemetry specification.
+export const ATTR_GEN_AI_USAGE_INPUT_TEXT_TOKENS = 'gen_ai.usage.input_text_tokens';
+export const ATTR_GEN_AI_USAGE_INPUT_AUDIO_TOKENS = 'gen_ai.usage.input_audio_tokens';
+export const ATTR_GEN_AI_USAGE_INPUT_CACHED_TOKENS = 'gen_ai.usage.input_cached_tokens';
+export const ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS = 'gen_ai.usage.output_text_tokens';
+export const ATTR_GEN_AI_USAGE_OUTPUT_AUDIO_TOKENS = 'gen_ai.usage.output_audio_tokens';
+
+// OpenTelemetry GenAI event names (for structured logging)
+export const EVENT_GEN_AI_SYSTEM_MESSAGE = 'gen_ai.system.message';
+export const EVENT_GEN_AI_USER_MESSAGE = 'gen_ai.user.message';
+export const EVENT_GEN_AI_ASSISTANT_MESSAGE = 'gen_ai.assistant.message';
+export const EVENT_GEN_AI_TOOL_MESSAGE = 'gen_ai.tool.message';
+export const EVENT_GEN_AI_CHOICE = 'gen_ai.choice';
+
+// Exception attributes
+export const ATTR_EXCEPTION_TRACE = 'exception.stacktrace';
+export const ATTR_EXCEPTION_TYPE = 'exception.type';
+export const ATTR_EXCEPTION_MESSAGE = 'exception.message';
+
+// Platform-specific attributes
+export const ATTR_LANGFUSE_COMPLETION_START_TIME = 'langfuse.observation.completion_start_time';

agents/src/telemetry/traces.ts

@@ -0,0 +1,153 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {
+  type Attributes,
+  type Context,
+  type Span,
+  type SpanOptions,
+  type Tracer,
+  type TracerProvider,
+  context as otelContext,
+  trace,
+} from '@opentelemetry/api';
+
+export interface SpanStartOptions {
+  /** Name of the span */
+  name: string;
+  /** Optional parent context to use for this span */
+  context?: Context;
+  /** Attributes to set on the span when it starts */
+  attributes?: Attributes;
+  /** Whether to end the span when the function exits (default: true) */
+  endOnExit?: boolean;
+}
+
+/**
+ * A dynamic tracer that allows the tracer provider to be changed at runtime.
+ */
+class DynamicTracer {
+  private tracerProvider: TracerProvider;
+  private tracer: Tracer;
+  private readonly instrumentingModuleName: string;
+
+  constructor(instrumentingModuleName: string) {
+    this.instrumentingModuleName = instrumentingModuleName;
+    this.tracerProvider = trace.getTracerProvider();
+    this.tracer = trace.getTracer(instrumentingModuleName);
+  }
+
+  /**
+   * Set a new tracer provider. This updates the underlying tracer instance.
+   * @param provider - The new tracer provider to use
+   */
+  setProvider(provider: TracerProvider): void {
+    this.tracerProvider = provider;
+    this.tracer = this.tracerProvider.getTracer(this.instrumentingModuleName);
+  }
+
+  /**
+   * Get the underlying OpenTelemetry tracer.
+   * Use this to access the full Tracer API when needed.
+   */
+  getTracer(): Tracer {
+    return this.tracer;
+  }
+
+  /**
+   * Start a span manually (without making it active).
+   * You must call span.end() when done.
+   *
+   * @param options - Span configuration including name
+   * @returns The created span
+   */
+  startSpan(options: SpanStartOptions): Span {
+    const ctx = options.context || otelContext.active();
+    const span = this.tracer.startSpan(
+      options.name,
+      {
+        attributes: options.attributes,
+      },
+      ctx,
+    );
+
+    return span;
+  }
+
+  /**
+   * Start a new span and make it active in the current context.
+   * The span will automatically be ended when the provided function completes (unless endOnExit=false).
+   *
+   * @param fn - The function to execute within the span context
+   * @param options - Span configuration including name
+   * @returns The result of the provided function
+   */
+  async startActiveSpan<T>(fn: (span: Span) => Promise<T>, options: SpanStartOptions): Promise<T> {
+    const ctx = options.context || otelContext.active();
+    const endOnExit = options.endOnExit === undefined ? true : options.endOnExit; // default true
+    const opts: SpanOptions = { attributes: options.attributes };
+
+    return new Promise((resolve, reject) => {
+      this.tracer.startActiveSpan(options.name, opts, ctx, async (span) => {
+        try {
+          const result = await fn(span);
+          resolve(result);
+        } catch (error) {
+          reject(error);
+        } finally {
+          if (endOnExit) {
+            span.end();
+          }
+        }
+      });
+    });
+  }
+
+  /**
+   * Synchronous version of startActiveSpan for non-async operations.
+   *
+   * @param fn - The function to execute within the span context
+   * @param options - Span configuration including name
+   * @returns The result of the provided function
+   */
+  startActiveSpanSync<T>(fn: (span: Span) => T, options: SpanStartOptions): T {
+    const ctx = options.context || otelContext.active();
+    const endOnExit = options.endOnExit === undefined ? true : options.endOnExit; // default true
+    const opts: SpanOptions = { attributes: options.attributes };
+
+    return this.tracer.startActiveSpan(options.name, opts, ctx, (span) => {
+      try {
+        return fn(span);
+      } finally {
+        if (endOnExit) {
+          span.end();
+        }
+      }
+    });
+  }
+}
+
+/**
+ * The global tracer instance used throughout the agents framework.
+ * This tracer can have its provider updated at runtime via setTracerProvider().
+ */
+export const tracer = new DynamicTracer('livekit-agents');
+
+/**
+ * Set the tracer provider for the livekit-agents framework.
+ * This should be called before agent session start if using custom tracer providers.
+ *
+ * @param provider - The tracer provider to use
+ *
+ * @example
+ * ```typescript
+ * import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
+ * import { setTracerProvider } from '@livekit/agents/telemetry';
+ *
+ * const provider = new NodeTracerProvider();
+ * setTracerProvider(provider);
+ * ```
+ */
+export function setTracerProvider(provider: TracerProvider): void {
+  tracer.setProvider(provider);
+}

agents/src/telemetry/utils.ts

@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { type Span, SpanStatusCode, context as otelContext, trace } from '@opentelemetry/api';
+import type { RealtimeModelMetrics } from '../metrics/base.js';
+import * as traceTypes from './trace_types.js';
+import { tracer } from './traces.js';
+
+export function recordException(span: Span, error: Error): void {
+  span.recordException(error);
+  span.setStatus({
+    code: SpanStatusCode.ERROR,
+    message: error.message,
+  });
+
+  // Set exception attributes for better visibility
+  // (in case the exception event is not rendered by the backend)
+  span.setAttributes({
+    [traceTypes.ATTR_EXCEPTION_TYPE]: error.constructor.name,
+    [traceTypes.ATTR_EXCEPTION_MESSAGE]: error.message,
+    [traceTypes.ATTR_EXCEPTION_TRACE]: error.stack || '',
+  });
+}
+
+export function recordRealtimeMetrics(span: Span, metrics: RealtimeModelMetrics): void {
+  const attrs: Record<string, string | number> = {
+    [traceTypes.ATTR_GEN_AI_REQUEST_MODEL]: metrics.label || 'unknown',
+    [traceTypes.ATTR_REALTIME_MODEL_METRICS]: JSON.stringify(metrics),
+    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.inputTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.outputTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TEXT_TOKENS]: metrics.inputTokenDetails.textTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_AUDIO_TOKENS]: metrics.inputTokenDetails.audioTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_CACHED_TOKENS]: metrics.inputTokenDetails.cachedTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS]: metrics.outputTokenDetails.textTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_AUDIO_TOKENS]: metrics.outputTokenDetails.audioTokens,
+  };
+
+  // Add LangFuse-specific completion start time if TTFT is available
+  if (metrics.ttftMs !== undefined && metrics.ttftMs !== -1) {
+    const completionStartTime = metrics.timestamp + metrics.ttftMs;
+    // Convert to UTC ISO string for LangFuse compatibility
+    const completionStartTimeUtc = new Date(completionStartTime).toISOString();
+    attrs[traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME] = completionStartTimeUtc;
+  }
+
+  if (span.isRecording()) {
+    span.setAttributes(attrs);
+  } else {
+    const currentContext = otelContext.active();
+    const spanContext = trace.setSpan(currentContext, span);
+
+    // Create a dedicated child span for orphaned metrics
+    tracer.getTracer().startActiveSpan('realtime_metrics', {}, spanContext, (child) => {
+      try {
+        child.setAttributes(attrs);
+      } finally {
+        child.end();
+      }
+    });
+  }
+}