From 7f9340a36ac82a311b1840b422ec8bb9eb4594c0 Mon Sep 17 00:00:00 2001 From: Necati Ozmen Date: Thu, 22 Jan 2026 13:27:28 +0300 Subject: [PATCH 1/5] feat: update observability documentation with new user and views sections --- website/observability/tracing/concept.md | 107 ---------------------- website/observability/tracing/overview.md | 70 +++----------- website/observability/tracing/users.md | 75 +++++++++++++++ website/observability/tracing/views.md | 65 +++++++++++++ website/sidebarsObservability.ts | 2 +- 5 files changed, 156 insertions(+), 163 deletions(-) delete mode 100644 website/observability/tracing/concept.md create mode 100644 website/observability/tracing/users.md create mode 100644 website/observability/tracing/views.md diff --git a/website/observability/tracing/concept.md b/website/observability/tracing/concept.md deleted file mode 100644 index 338c1c7f4..000000000 --- a/website/observability/tracing/concept.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -title: Concept ---- - -In VoltOps, a trace is a record of the entire process from a user's request to the system's response. Each trace captures all steps, decisions, and data flow from the beginning to the end of a single user interaction. - -:::tip Solves the black-box problem -Traces eliminate the "black box" nature of AI applications by providing complete transparency and enabling you to manage your systems with confidence. -::: - -VoltOps provides **n8n-style observability** with a visual canvas interface that makes complex AI workflows intuitive to understand. - -:::note The n8n-inspired canvas visualization offers several key advantages - -- **Visual Workflow Mapping**: See your entire AI workflow as connected nodes, making it easy to understand data flow and decision points -- **Intuitive Debugging**: Quickly identify where issues occur by visually following the execution path through your AI agents and tools -- **Non-Technical Accessibility**: Enable stakeholders without deep technical knowledge to understand and analyze AI system behavior -- **Real-time Monitoring**: Watch your AI workflows execute in real-time with live status updates on the visual canvas -- **Simplified Complexity**: Transform complex, multi-step AI processes into clear, digestible visual representations - ::: - -## Core Trace Features - -
- -```mermaid -%%{init: {'theme':'base', 'themeVariables': {'primaryColor':'#10b981', 'primaryTextColor':'#ffffff', 'primaryBorderColor':'#10b981', 'lineColor':'#10b981', 'sectionBkgColor':'#10b981', 'altSectionBkgColor':'#059669', 'gridColor':'#10b981', 'secondaryColor':'#10b981', 'tertiaryColor':'#ffffff'}}}%% -sequenceDiagram - participant User - participant App as LLM Application - participant VoltOps as VoltOps Observability - participant LLM as LLM/Tools/DB - - User->>App: Send Request - App->>VoltOps: Initialize Trace - - App->>LLM: AI Operations (Prompts, Tools, Memory) - VoltOps->>VoltOps: Track: Tokens, Cost, Latency, Status - LLM->>App: Responses & Data - - App->>User: Send Response - App->>VoltOps: Complete Trace - - Note over VoltOps: Real-time Analytics & Monitoring -``` - -
- -**Unique Session Identity:** -Each trace is identified with a unique session ID and maintains complete context throughout the interaction lifecycle. - -**Flexible Organization:** -Traces support tagging and metadata for easy categorization and filtering across your AI workflows. - -**Real-time Visibility:** -VoltOps captures and visualizes traces in real-time, enabling instant monitoring and issue detection. - -**Hierarchical Structure:** -Agents, tools, memory operations, and retrievers are organized in meaningful hierarchies that reflect your AI application's logic. - -**Complete Lifecycle Tracking:** -From start to finish, traces monitor status changes, performance metrics, and outcomes. - -
- - **Information Captured in Traces** - - -### Session and Context Data - -- **User Identity**: User ID, profile information, and preferences -- **Conversation Context**: Conversation ID, history references, and session state -- **Input/Output Flow**: Original requests, system responses, and data transformations -- **Custom Metadata**: Priority levels, source channels, departments, and domain-specific information - -### Performance and Cost Metrics - -- **Token Usage**: Prompt tokens, completion tokens, and total consumption -- **Timing Data**: Processing times, response latency, and execution duration -- **Cost Analysis**: API call costs and resource consumption - -### Error and Status Information - -- **Status Tracking**: Running, completed, or error states with timestamps -- **Error Details**: Messages, codes, and failure stages -- **Success Metrics**: Completion rates, confidence scores, and quality indicators -- **Audit Trail**: Decision points and state transitions - -
- -## VoltOps vs General Observability Concepts - -VoltOps tracing builds upon established observability concepts but adapts them specifically for AI agent workflows. Understanding how VoltOps concepts map to general observability terms helps you leverage existing knowledge while working with AI-native structures. - -### Concept Mapping - -**VoltOps Trace ≡ General Trace** -Both represent a complete user interaction session from request to response, maintaining unique identifiers, timestamps, and hierarchical organization. - -**VoltOps Agent ≈ Event + Trace** -An agent functions as both a span (tracking execution and hierarchy) and a generation (capturing AI model interactions with parameters, token usage, and responses). - -**VoltOps Tools/Memory/Retrievers ≈ Events** -These operations behave like traditional spans, tracking execution time, success/error states, input/output data, and nested hierarchies. - -**VoltOps Success/Error Events ≈ Discrete Events** -Status changes function as timestamped events marking state transitions and providing audit trails. diff --git a/website/observability/tracing/overview.md b/website/observability/tracing/overview.md index 358e1c022..8392155f4 100644 --- a/website/observability/tracing/overview.md +++ b/website/observability/tracing/overview.md @@ -4,7 +4,7 @@ title: Overview # VoltOps Tracing -Tracing is the process of recording and visualizing the execution path of your AI applications in real-time. It shows you exactly what your AI agents are doing, which tools they're using, and how data flows through your system from start to finish. +Tracing shows you exactly what happened in a single user request, end to end. Use it to find where a flow failed, which step is slow, or why a user got a specific response. It is most useful for agent debugging, performance and cost tuning, QA checks, and audit trails. -## Why Tracing Matters for LLM Applications +
+
-
- -Understanding Complex AI Workflows - +When you select a trace from the list, you can open its [detailed views](https://voltagent.dev/observability-docs/tracing/views/). -Large Language Model applications often involve complex, multi-step processes that can be difficult to debug and optimize. Unlike traditional applications with predictable execution paths, LLM apps feature: +## Trace Filters -- **Dynamic decision-making**: AI agents make context-dependent choices that vary between runs -- **Multi-step reasoning**: Complex tasks are broken down into multiple sequential or parallel operations -- **Tool integration**: AI agents interact with external APIs, databases, and services -- **Non-deterministic behavior**: The same input can produce different execution paths +The trace list includes filters so you can narrow down the exact runs you care about before opening details. -
+![Trace filters overview](https://cdn.voltagent.dev/docs/observability/tracing/overview-trace-filters.gif) -
- -Key Benefits of Tracing - - -**Debug with Confidence** - -- Identify exactly where errors occur in your AI workflow -- Understand why certain decisions were made by your AI agents -- Track the flow of data through complex processing chains -- Pinpoint performance bottlenecks in real-time - -**Monitor Performance** - -- Track response times for each component of your AI system -- Monitor token usage and costs across different LLM calls -- Identify slow operations that impact user experience - -**Optimize Your AI Applications** - -- Analyze which tools and prompts perform best -- Identify redundant or inefficient processing steps -- Optimize prompt engineering based on actual execution data -- Fine-tune your AI workflows for better performance - -**Collaborate Effectively** - -- Share detailed traces with team members for debugging -- Document AI behavior patterns for future reference -- Enable non-technical stakeholders to understand AI decision-making -- Create reproducible test cases from real execution traces - -
- -### Common Use Cases - -- **Agent Debugging**: When your AI agent produces unexpected results, tracing shows exactly what happened -- **Performance Optimization**: Identify which LLM calls or tool executions are taking too long -- **Cost Analysis**: Track token usage and API costs across your entire application -- **Quality Assurance**: Verify that your AI workflows behave consistently across different scenarios -- **Compliance**: Maintain audit trails of AI decision-making for regulatory requirements - -Tracing transforms the black box of AI applications into a transparent, observable system that you can understand, debug, and optimize with confidence. +- **Status**: isolate failed or retry-heavy runs. +- **Agent ID**: compare behavior across agents. +- **Entity type**: focus on agent/tool/llm steps at scale. +- **Token usage**: find expensive runs by input/output size. +- **Cost**: spot budget spikes quickly. +- **Duration**: isolate slow traces and latency outliers. +- **Feedback source / key**: review user or model feedback by source. +- **User ID / Conversation ID**: drill into a specific user journey. diff --git a/website/observability/tracing/users.md b/website/observability/tracing/users.md new file mode 100644 index 000000000..2341115be --- /dev/null +++ b/website/observability/tracing/users.md @@ -0,0 +1,75 @@ +--- +title: Users +--- + +# Users + +The Users page groups traces by user so you can quickly see who is affected by errors, high latency, or unusual cost and then jump into details. + +:::note Add visuals +Add the Users list screenshot or video here. +::: + +## Users List + +Each row represents a user. Use this list to find users that need attention and open their detail view. + +## User Detail View + +The detail page follows the same top-to-bottom layout as the UI. + +### Profile and Metadata + +Shows user identity and metadata (plan, tier, region) so you can segment behavior and compare cohorts. Use this to spot plan-based issues or regional latency differences. + +:::note Add visuals +Add the user header and metadata screenshot here. +::: + +### Key Metrics + +Gives a quick read of volume, success rate, latency, and cost. Use this to decide whether to investigate quality, performance, or spend first. + +:::note Add visuals +Add the metrics cards screenshot here. +::: + +### Trends and Model Usage + +Time-range charts show changes over time; model usage shows which model versions the user saw. This helps connect regressions to releases or model switches. + +:::note Add visuals +Add the trends and model usage charts here. +::: + +### User Feedback (Optional) + +Shows ratings and comments with the related traces so you can compare behavior with user feedback. Use it to validate fixes and track perceived quality. + +:::note Add visuals +Add the user feedback section screenshots here. +::: + +Each chart answers a specific question: + +- **Feedback Summary**: Quick snapshot of volume, average score, and coverage. +- **Feedback Trends**: How feedback volume and sentiment change over time. +- **Score Distribution**: Where scores cluster to spot polarization or drift. +- **Top Feedback Keys**: Which feedback signals dominate for this user. +- **Comment Momentum**: When users leave comments, not just scores. + +### Conversations + +Groups traces into user journeys; selecting one filters the trace list below. This is useful when a problem spans multiple steps instead of a single trace. + +:::note Add visuals +Add the conversations section screenshot here. +::: + +### Traces + +Lists all runs for the user; filtered by conversation when selected. + +:::note Add visuals +Add the user trace table screenshot here. +::: diff --git a/website/observability/tracing/views.md b/website/observability/tracing/views.md new file mode 100644 index 000000000..2e05ce9d9 --- /dev/null +++ b/website/observability/tracing/views.md @@ -0,0 +1,65 @@ +--- +title: Views +--- + +# Trace Views + +When you open a trace, the drawer shows three tabs. Each tab answers a different question. + +## Waterfall + +Use Waterfall to understand timing and execution order. It shows spans as a hierarchy so you can see where time is spent, how retries fan out, and which step failed first. + + + +
+
+ +Use Waterfall to answer three core questions: what failed, what was slow, and what the model actually saw. The details panel gives the context you need to confirm tool usage, memory behavior, and LLM settings without digging through logs. + +Key signals to check: + +- **Failure path**: error badges + child spans to see the first break and its causes. +- **Latency hotspots**: duration bars and timing to find the slowest steps. +- **Model context**: input/output and UI vs model messages to verify prompts and responses. +- **Configuration**: LLM config, memory config, and knowledge base usage for this span. + +## Logs + +Use Logs to review structured events tied to the trace and span context. This is the fastest way to inspect errors and metadata without opening each span. + + + +
+
+ +Logs are best when you need a fast answer to “what happened and why.” Start with errors, then follow the trace/span context to the exact step, and use attributes to see the payload that caused it. + +## Feedback + +Use Feedback to capture human signals for a trace. It helps you compare agent behavior with user sentiment and track quality over time. + + + +What to look at: + +- **Feedback history**: see if this trace has already been rated and by which source. +- **Active key**: understand which signal you are recording (for example, satisfaction). +- **Helpful / Not helpful**: capture a simple quality signal for fast triage. +- **Comments**: add short context that explains why a trace felt good or bad. + +:::note Add visuals +Add the Feedback screenshot or GIF here. +::: diff --git a/website/sidebarsObservability.ts b/website/sidebarsObservability.ts index 455b9202a..0230ec4be 100644 --- a/website/sidebarsObservability.ts +++ b/website/sidebarsObservability.ts @@ -28,7 +28,7 @@ const sidebars: SidebarsConfig = { { type: "category", label: "Tracing", - items: ["tracing/overview", "tracing/concept"], + items: ["tracing/overview", "tracing/views", "tracing/users"], }, { type: "doc", From 86ba17575067370959af78eef4b805f472a1f64f Mon Sep 17 00:00:00 2001 From: Necati Ozmen Date: Thu, 22 Jan 2026 18:07:06 +0300 Subject: [PATCH 2/5] feat: enhance observability documentation with improved clarity and structure --- website/observability/tracing/overview.md | 30 ++++++--- website/observability/tracing/users.md | 81 ++++++++--------------- website/observability/tracing/views.md | 33 ++++----- 3 files changed, 63 insertions(+), 81 deletions(-) diff --git a/website/observability/tracing/overview.md b/website/observability/tracing/overview.md index 8392155f4..21420dbda 100644 --- a/website/observability/tracing/overview.md +++ b/website/observability/tracing/overview.md @@ -4,7 +4,11 @@ title: Overview # VoltOps Tracing -Tracing shows you exactly what happened in a single user request, end to end. Use it to find where a flow failed, which step is slow, or why a user got a specific response. It is most useful for agent debugging, performance and cost tuning, QA checks, and audit trails. +Tracing shows how a single user request runs from start to finish. As a developer, you use it to answer: + +- Why did the agent respond this way? +- Which step failed? +- Where did latency or cost spike?