diff --git a/.changeset/patch-add-logs-output-guardrail.md b/.changeset/patch-add-logs-output-guardrail.md new file mode 100644 index 00000000000..c956bd762b1 --- /dev/null +++ b/.changeset/patch-add-logs-output-guardrail.md @@ -0,0 +1,5 @@ +--- +"gh-aw": patch +--- + +Add 10KB output size guardrail to MCP server logs command diff --git a/.github/workflows/go.mod b/.github/workflows/go.mod index 0eed60d3481..189adb44024 100644 --- a/.github/workflows/go.mod +++ b/.github/workflows/go.mod @@ -1,5 +1,3 @@ module github.com/githubnext/gh-aw-workflows-deps -go 1.21 -require ( -) +go 1.21 diff --git a/.gitignore b/.gitignore index dfe67e24b77..c4c44e709f5 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,6 @@ __pycache__/ *.py[cod] *$py.class *.pyc + +# Manual test scripts +test_*.sh diff --git a/pkg/cli/mcp_logs_guardrail.go b/pkg/cli/mcp_logs_guardrail.go new file mode 100644 index 00000000000..3794bb3c706 --- /dev/null +++ b/pkg/cli/mcp_logs_guardrail.go @@ -0,0 +1,219 @@ +package cli + +import ( + "encoding/json" + "fmt" + "strings" +) + +const ( + // DefaultMaxMCPLogsOutputTokens is the default maximum number of tokens for MCP logs output + // before triggering the guardrail (12000 tokens) + DefaultMaxMCPLogsOutputTokens = 12000 + + // CharsPerToken is the approximate number of characters per token + // Using OpenAI's rule of thumb: ~4 characters per token + CharsPerToken = 4 +) + +// MCPLogsGuardrailResponse represents the response when output is too large +type MCPLogsGuardrailResponse struct { + Message string `json:"message"` + OutputTokens int `json:"output_tokens"` + OutputSizeLimit int `json:"output_size_limit"` + Schema LogsDataSchema `json:"schema"` + SuggestedQueries []SuggestedJqQuery `json:"suggested_queries"` +} + +// LogsDataSchema describes the structure of the full logs output +type LogsDataSchema struct { + Description string `json:"description"` + Type string `json:"type"` + Fields map[string]SchemaField `json:"fields"` +} + +// SchemaField describes a field in the schema +type SchemaField struct { + Type string `json:"type"` + Description string `json:"description"` +} + +// SuggestedJqQuery represents a suggested jq filter +type SuggestedJqQuery struct { + Description string `json:"description"` + Query string `json:"query"` + Example string `json:"example,omitempty"` +} + +// estimateTokens estimates the number of tokens in a string +// Using the approximation: ~4 characters per token +func estimateTokens(text string) int { + return len(text) / CharsPerToken +} + +// checkLogsOutputSize checks if the logs output exceeds the token limit +// and returns a guardrail response if it does +func checkLogsOutputSize(outputStr string, maxTokens int) (string, bool) { + if maxTokens == 0 { + maxTokens = DefaultMaxMCPLogsOutputTokens + } + + outputTokens := estimateTokens(outputStr) + + if outputTokens <= maxTokens { + return outputStr, false + } + + // Generate guardrail response + guardrail := MCPLogsGuardrailResponse{ + Message: fmt.Sprintf( + "⚠️ Output size (%d tokens) exceeds the limit (%d tokens). "+ + "To reduce output size, use the 'jq' parameter with one of the suggested queries below.", + outputTokens, + maxTokens, + ), + OutputTokens: outputTokens, + OutputSizeLimit: maxTokens, + Schema: getLogsDataSchema(), + SuggestedQueries: getSuggestedJqQueries(), + } + + // Marshal to JSON + guardrailJSON, err := json.MarshalIndent(guardrail, "", " ") + if err != nil { + // Fallback to simple text message if JSON marshaling fails + return fmt.Sprintf( + "Output size (%d tokens) exceeds the limit (%d tokens). "+ + "Please use the 'jq' parameter to filter the output.", + outputTokens, + maxTokens, + ), true + } + + return string(guardrailJSON), true +} + +// getLogsDataSchema returns the schema for LogsData +func getLogsDataSchema() LogsDataSchema { + return LogsDataSchema{ + Description: "Complete structured data for workflow logs", + Type: "object", + Fields: map[string]SchemaField{ + "summary": { + Type: "object", + Description: "Aggregate metrics across all runs (total_runs, total_duration, total_tokens, total_cost, total_turns, total_errors, total_warnings, total_missing_tools)", + }, + "runs": { + Type: "array", + Description: "Array of workflow run data (database_id, workflow_name, agent, status, conclusion, duration, token_usage, estimated_cost, turns, error_count, warning_count, missing_tool_count, created_at, url, logs_path, event, branch)", + }, + "tool_usage": { + Type: "array", + Description: "Tool usage statistics (name, total_calls, runs, max_output_size, max_duration)", + }, + "errors_and_warnings": { + Type: "array", + Description: "Error and warning summaries (type, message, count, engine, run_id, run_url, workflow_name, pattern_id)", + }, + "missing_tools": { + Type: "array", + Description: "Missing tool reports (tool, count, workflows, first_reason, run_ids)", + }, + "mcp_failures": { + Type: "array", + Description: "MCP server failure summaries (server_name, count, workflows, run_ids)", + }, + "access_log": { + Type: "object", + Description: "Access log analysis (total_requests, allowed_count, denied_count, allowed_domains, denied_domains, by_workflow)", + }, + "firewall_log": { + Type: "object", + Description: "Firewall log analysis (total_requests, allowed_requests, denied_requests, allowed_domains, denied_domains, requests_by_domain, by_workflow)", + }, + "continuation": { + Type: "object", + Description: "Parameters to continue querying when timeout is reached (message, workflow_name, count, start_date, end_date, engine, branch, after_run_id, before_run_id, timeout)", + }, + "logs_location": { + Type: "string", + Description: "File system path where logs were downloaded", + }, + }, + } +} + +// getSuggestedJqQueries returns a list of useful jq queries to reduce output +func getSuggestedJqQueries() []SuggestedJqQuery { + return []SuggestedJqQuery{ + { + Description: "Get only the summary statistics", + Query: ".summary", + Example: "Use jq parameter: \".summary\"", + }, + { + Description: "Get list of run IDs and workflow names", + Query: ".runs | map({database_id, workflow_name, status})", + Example: "Use jq parameter: \".runs | map({database_id, workflow_name, status})\"", + }, + { + Description: "Get only failed runs", + Query: ".runs | map(select(.conclusion == \"failure\"))", + Example: "Use jq parameter: \".runs | map(select(.conclusion == \\\"failure\\\"))\"", + }, + { + Description: "Get summary with first 5 runs", + Query: "{summary, runs: .runs[:5]}", + Example: "Use jq parameter: \"{summary, runs: .runs[:5]}\"", + }, + { + Description: "Get only error and warning summaries", + Query: "{errors_and_warnings, missing_tools, mcp_failures}", + Example: "Use jq parameter: \"{errors_and_warnings, missing_tools, mcp_failures}\"", + }, + { + Description: "Get tool usage statistics", + Query: ".tool_usage", + Example: "Use jq parameter: \".tool_usage\"", + }, + { + Description: "Get runs with high token usage (>10k tokens)", + Query: ".runs | map(select(.token_usage > 10000))", + Example: "Use jq parameter: \".runs | map(select(.token_usage > 10000))\"", + }, + { + Description: "Get runs from specific workflow", + Query: ".runs | map(select(.workflow_name == \"YOUR_WORKFLOW_NAME\"))", + Example: "Use jq parameter: \".runs | map(select(.workflow_name == \\\"YOUR_WORKFLOW_NAME\\\"))\"", + }, + } +} + +// formatGuardrailMessage creates a user-friendly text message from the guardrail response +func formatGuardrailMessage(guardrail MCPLogsGuardrailResponse) string { + var builder strings.Builder + + builder.WriteString(guardrail.Message) + builder.WriteString("\n\n") + + builder.WriteString("📋 Output Schema:\n") + builder.WriteString(fmt.Sprintf(" Type: %s\n", guardrail.Schema.Type)) + builder.WriteString(fmt.Sprintf(" Description: %s\n\n", guardrail.Schema.Description)) + + builder.WriteString("Available fields:\n") + for field, schema := range guardrail.Schema.Fields { + builder.WriteString(fmt.Sprintf(" - %s (%s): %s\n", field, schema.Type, schema.Description)) + } + + builder.WriteString("\n💡 Suggested jq Queries:\n") + for i, query := range guardrail.SuggestedQueries { + builder.WriteString(fmt.Sprintf(" %d. %s\n", i+1, query.Description)) + builder.WriteString(fmt.Sprintf(" Query: %s\n", query.Query)) + if query.Example != "" { + builder.WriteString(fmt.Sprintf(" %s\n", query.Example)) + } + builder.WriteString("\n") + } + + return builder.String() +} diff --git a/pkg/cli/mcp_logs_guardrail_integration_test.go b/pkg/cli/mcp_logs_guardrail_integration_test.go new file mode 100644 index 00000000000..c59fa5de2f4 --- /dev/null +++ b/pkg/cli/mcp_logs_guardrail_integration_test.go @@ -0,0 +1,115 @@ +//go:build integration + +package cli + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// TestMCPServer_LogsGuardrail tests the output size guardrail on the logs tool +func TestMCPServer_LogsGuardrail(t *testing.T) { + // Skip if the binary doesn't exist + binaryPath := "../../gh-aw" + if _, err := os.Stat(binaryPath); os.IsNotExist(err) { + t.Skip("Skipping test: gh-aw binary not found. Run 'make build' first.") + } + + // Create MCP client + client := mcp.NewClient(&mcp.Implementation{ + Name: "test-client", + Version: "1.0.0", + }, nil) + + // Start the MCP server as a subprocess + serverCmd := exec.Command(binaryPath, "mcp-server") + transport := &mcp.CommandTransport{Command: serverCmd} + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + session, err := client.Connect(ctx, transport, nil) + if err != nil { + t.Fatalf("Failed to connect to MCP server: %v", err) + } + defer session.Close() + + t.Run("small output passes through normally", func(t *testing.T) { + // This test is informational - it documents expected behavior + // In a real environment with workflows, calling logs with count=1 and jq filter + // should produce output small enough to pass through without triggering guardrail + t.Skip("Skipping real logs call test - requires repository with workflows") + }) + + t.Run("guardrail provides schema and suggestions", func(t *testing.T) { + // We can't easily trigger the guardrail in a real scenario without + // having a large amount of logs, so this test documents the expected + // behavior and structure of the guardrail response + + // Test that checkLogsOutputSize produces the expected structure + // Default limit is 12000 tokens = 48000 characters + // Use 50000 to safely exceed the limit + largeOutput := strings.Repeat("x", 50000) + guardrailJSON, triggered := checkLogsOutputSize(largeOutput, 0) + + if !triggered { + t.Fatal("Guardrail should be triggered for large output") + } + + // Parse the guardrail response + var guardrail MCPLogsGuardrailResponse + if err := json.Unmarshal([]byte(guardrailJSON), &guardrail); err != nil { + t.Fatalf("Guardrail response should be valid JSON: %v", err) + } + + // Verify guardrail has all expected components + if guardrail.Message == "" { + t.Error("Guardrail should have a message") + } + + if !strings.Contains(guardrail.Message, "exceeds the limit") { + t.Error("Message should explain the issue") + } + + if !strings.Contains(guardrail.Message, "tokens") { + t.Error("Message should mention tokens") + } + + if guardrail.Schema.Type != "object" { + t.Error("Schema should be object type") + } + + if len(guardrail.Schema.Fields) == 0 { + t.Error("Schema should have fields") + } + + if len(guardrail.SuggestedQueries) == 0 { + t.Error("Should have suggested queries") + } + + // Verify some expected fields are in the schema + expectedFields := []string{"summary", "runs", "tool_usage", "errors_and_warnings"} + for _, field := range expectedFields { + if _, ok := guardrail.Schema.Fields[field]; !ok { + t.Errorf("Schema should include field '%s'", field) + } + } + + // Verify suggested queries have proper structure + for i, query := range guardrail.SuggestedQueries { + if query.Description == "" { + t.Errorf("Query %d should have description", i) + } + if query.Query == "" { + t.Errorf("Query %d should have query string", i) + } + } + }) +} diff --git a/pkg/cli/mcp_logs_guardrail_test.go b/pkg/cli/mcp_logs_guardrail_test.go new file mode 100644 index 00000000000..493bb89970e --- /dev/null +++ b/pkg/cli/mcp_logs_guardrail_test.go @@ -0,0 +1,303 @@ +package cli + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestCheckLogsOutputSize_SmallOutput(t *testing.T) { + // Create a small output (less than default token limit) + smallOutput := `{"summary": {"total_runs": 1}, "runs": []}` + + result, triggered := checkLogsOutputSize(smallOutput, 0) + + if triggered { + t.Error("Guardrail should not be triggered for small output") + } + + if result != smallOutput { + t.Error("Output should be unchanged for small output") + } +} + +func TestCheckLogsOutputSize_LargeOutput(t *testing.T) { + // Create a large output (more than default token limit) + // Default is 12000 tokens, which is ~48000 characters + // Use 50000 to be safely over the limit + largeOutput := strings.Repeat("x", 50000) + + result, triggered := checkLogsOutputSize(largeOutput, 0) + + if !triggered { + t.Error("Guardrail should be triggered for large output") + } + + if result == largeOutput { + t.Error("Output should be replaced with guardrail response for large output") + } + + // Verify the result contains a valid JSON guardrail response + var guardrail MCPLogsGuardrailResponse + if err := json.Unmarshal([]byte(result), &guardrail); err != nil { + t.Errorf("Guardrail response should be valid JSON: %v", err) + } + + // Verify guardrail response structure + if guardrail.Message == "" { + t.Error("Guardrail response should have a message") + } + + expectedTokens := estimateTokens(largeOutput) + if guardrail.OutputTokens != expectedTokens { + t.Errorf("Guardrail should report correct output tokens: expected %d, got %d", expectedTokens, guardrail.OutputTokens) + } + + if guardrail.OutputSizeLimit != DefaultMaxMCPLogsOutputTokens { + t.Errorf("Guardrail should report correct limit: expected %d, got %d", DefaultMaxMCPLogsOutputTokens, guardrail.OutputSizeLimit) + } + + if len(guardrail.SuggestedQueries) == 0 { + t.Error("Guardrail response should have suggested queries") + } + + if len(guardrail.Schema.Fields) == 0 { + t.Error("Guardrail response should have schema fields") + } +} + +func TestCheckLogsOutputSize_ExactLimit(t *testing.T) { + // Create output exactly at the limit + // 12000 tokens = 48000 characters + exactOutput := strings.Repeat("x", 48000) + + result, triggered := checkLogsOutputSize(exactOutput, 0) + + if triggered { + t.Error("Guardrail should not be triggered for output at exact limit") + } + + if result != exactOutput { + t.Error("Output should be unchanged for output at exact limit") + } +} + +func TestCheckLogsOutputSize_JustOverLimit(t *testing.T) { + // Create output just over the limit (12000 tokens + 1 token) + // 12001 tokens = 48004+ characters + overOutput := strings.Repeat("x", 48005) + + _, triggered := checkLogsOutputSize(overOutput, 0) + + if !triggered { + t.Error("Guardrail should be triggered for output just over limit") + } +} + +func TestCheckLogsOutputSize_CustomLimit(t *testing.T) { + // Test with custom token limit + customLimit := 100 + // 100 tokens = 400 characters, so use 500 to exceed + largeOutput := strings.Repeat("x", 500) + + result, triggered := checkLogsOutputSize(largeOutput, customLimit) + + if !triggered { + t.Error("Guardrail should be triggered when exceeding custom limit") + } + + var guardrail MCPLogsGuardrailResponse + if err := json.Unmarshal([]byte(result), &guardrail); err != nil { + t.Errorf("Guardrail response should be valid JSON: %v", err) + } + + if guardrail.OutputSizeLimit != customLimit { + t.Errorf("Guardrail should report custom limit: expected %d, got %d", customLimit, guardrail.OutputSizeLimit) + } +} + +func TestGetLogsDataSchema(t *testing.T) { + schema := getLogsDataSchema() + + // Verify basic schema structure + if schema.Type != "object" { + t.Errorf("Schema type should be 'object', got '%s'", schema.Type) + } + + if schema.Description == "" { + t.Error("Schema should have a description") + } + + // Verify expected fields are present + expectedFields := []string{ + "summary", + "runs", + "tool_usage", + "errors_and_warnings", + "missing_tools", + "mcp_failures", + "access_log", + "firewall_log", + "continuation", + "logs_location", + } + + for _, field := range expectedFields { + if _, ok := schema.Fields[field]; !ok { + t.Errorf("Schema should have field '%s'", field) + } + } + + // Verify each field has type and description + for fieldName, field := range schema.Fields { + if field.Type == "" { + t.Errorf("Field '%s' should have a type", fieldName) + } + if field.Description == "" { + t.Errorf("Field '%s' should have a description", fieldName) + } + } +} + +func TestGetSuggestedJqQueries(t *testing.T) { + queries := getSuggestedJqQueries() + + if len(queries) == 0 { + t.Error("Should have at least one suggested query") + } + + // Verify each query has required fields + for i, query := range queries { + if query.Description == "" { + t.Errorf("Query %d should have a description", i) + } + if query.Query == "" { + t.Errorf("Query %d should have a query string", i) + } + } + + // Verify we have some common useful queries + hasBasicQueries := false + for _, query := range queries { + if strings.Contains(query.Query, ".summary") { + hasBasicQueries = true + break + } + } + + if !hasBasicQueries { + t.Error("Should have basic summary query in suggestions") + } +} + +func TestFormatGuardrailMessage(t *testing.T) { + guardrail := MCPLogsGuardrailResponse{ + Message: "Test message", + OutputTokens: 15000, + OutputSizeLimit: DefaultMaxMCPLogsOutputTokens, + Schema: getLogsDataSchema(), + SuggestedQueries: getSuggestedJqQueries(), + } + + message := formatGuardrailMessage(guardrail) + + // Verify message contains key components + if !strings.Contains(message, "Test message") { + t.Error("Formatted message should contain the original message") + } + + if !strings.Contains(message, "Output Schema") { + t.Error("Formatted message should contain schema section") + } + + if !strings.Contains(message, "Suggested jq Queries") { + t.Error("Formatted message should contain suggested queries section") + } + + // Verify it mentions some fields + if !strings.Contains(message, "summary") { + t.Error("Formatted message should mention 'summary' field") + } +} + +func TestGuardrailResponseJSON(t *testing.T) { + // Create a large output to trigger guardrail + // Default limit is 12000 tokens = 48000 characters + largeOutput := strings.Repeat("x", 96000) + + result, triggered := checkLogsOutputSize(largeOutput, 0) + + if !triggered { + t.Fatal("Guardrail should be triggered") + } + + // Parse the JSON response + var guardrail MCPLogsGuardrailResponse + if err := json.Unmarshal([]byte(result), &guardrail); err != nil { + t.Fatalf("Should return valid JSON: %v", err) + } + + // Verify the JSON structure is complete and valid + if guardrail.Message == "" { + t.Error("JSON should have message field") + } + + if guardrail.OutputTokens == 0 { + t.Error("JSON should have output_tokens field") + } + + if guardrail.OutputSizeLimit == 0 { + t.Error("JSON should have output_size_limit field") + } + + if guardrail.Schema.Type == "" { + t.Error("JSON should have schema.type field") + } + + if len(guardrail.Schema.Fields) == 0 { + t.Error("JSON should have schema.fields") + } + + if len(guardrail.SuggestedQueries) == 0 { + t.Error("JSON should have suggested_queries") + } + + // Verify each suggested query has the expected fields + for i, query := range guardrail.SuggestedQueries { + if query.Description == "" { + t.Errorf("Query %d should have description in JSON", i) + } + if query.Query == "" { + t.Errorf("Query %d should have query in JSON", i) + } + } +} + +func TestDefaultMaxTokensConstant(t *testing.T) { + // Verify the constant is set to expected value (12000 tokens) + expected := 12000 + if DefaultMaxMCPLogsOutputTokens != expected { + t.Errorf("DefaultMaxMCPLogsOutputTokens should be %d tokens, got %d", expected, DefaultMaxMCPLogsOutputTokens) + } +} + +func TestEstimateTokens(t *testing.T) { + // Test token estimation + tests := []struct { + text string + expectedTokens int + }{ + {"", 0}, + {"x", 0}, // 1 char / 4 = 0 + {"xxxx", 1}, // 4 chars / 4 = 1 + {"xxxxxxxx", 2}, // 8 chars / 4 = 2 + {strings.Repeat("x", 400), 100}, // 400 chars / 4 = 100 + } + + for _, tt := range tests { + got := estimateTokens(tt.text) + if got != tt.expectedTokens { + t.Errorf("estimateTokens(%q) = %d, want %d", tt.text, got, tt.expectedTokens) + } + } +} diff --git a/pkg/cli/mcp_server.go b/pkg/cli/mcp_server.go index ebe6228be29..4e27166bcec 100644 --- a/pkg/cli/mcp_server.go +++ b/pkg/cli/mcp_server.go @@ -233,6 +233,7 @@ Note: Output can be filtered using the jq parameter.`, BeforeRunID int64 `json:"before_run_id,omitempty" jsonschema:"Filter runs with database ID before this value (exclusive)"` Timeout int `json:"timeout,omitempty" jsonschema:"Maximum time in seconds to spend downloading logs (default: 50 for MCP server)"` JqFilter string `json:"jq,omitempty" jsonschema:"Optional jq filter to apply to JSON output"` + MaxTokens int `json:"max_tokens,omitempty" jsonschema:"Maximum number of tokens in output before triggering guardrail (default: 12000)"` } mcp.AddTool(server, &mcp.Tool{ Name: "logs", @@ -243,7 +244,14 @@ a "continuation" field will be present in the response with updated parameters t Check for the presence of the continuation field to determine if there are more logs available. The continuation field includes all necessary parameters (before_run_id, etc.) to resume fetching from where -the previous request stopped due to timeout.`, +the previous request stopped due to timeout. + +⚠️ Output Size Guardrail: If the output exceeds the token limit (default: 12000 tokens), the tool will +return a schema description and suggested jq filters instead of the full output. Use the 'jq' parameter +to filter the output to a manageable size, or adjust the 'max_tokens' parameter. Common filters include: + - .summary (get only summary statistics) + - .runs[:5] (get first 5 runs) + - .runs | map(select(.conclusion == "failure")) (get only failed runs)`, }, func(ctx context.Context, req *mcp.CallToolRequest, args logsArgs) (*mcp.CallToolResult, any, error) { // Build command arguments // Force output directory to /tmp/gh-aw/aw-mcp/logs for MCP server @@ -309,9 +317,12 @@ the previous request stopped due to timeout.`, outputStr = filteredOutput } + // Check output size and apply guardrail if needed + finalOutput, _ := checkLogsOutputSize(outputStr, args.MaxTokens) + return &mcp.CallToolResult{ Content: []mcp.Content{ - &mcp.TextContent{Text: outputStr}, + &mcp.TextContent{Text: finalOutput}, }, }, nil, nil }) diff --git a/specs/MCP_LOGS_GUARDRAIL.md b/specs/MCP_LOGS_GUARDRAIL.md new file mode 100644 index 00000000000..6760e84fb4d --- /dev/null +++ b/specs/MCP_LOGS_GUARDRAIL.md @@ -0,0 +1,238 @@ +# MCP Server Logs Guardrail + +This document describes the output size guardrail implemented for the MCP server's `logs` command. + +## Problem + +When using the MCP server to fetch workflow logs, the output can become very large, especially when: +- Fetching logs for many workflow runs +- Runs contain extensive tool usage data +- Multiple workflows are being analyzed + +Large outputs can: +- Exceed token limits in AI models +- Cause performance issues in MCP clients +- Make it difficult to process and understand the data + +## Solution + +The MCP server `logs` command now includes an automatic guardrail that: + +1. **Checks output size** before returning results +2. **Triggers at 12000 tokens** (default, configurable) +3. **Returns helpful guidance** instead of large payloads + +## How It Works + +### Normal Operation (Output ≤ Token Limit) + +When the output is within the token limit, the command returns the full JSON data as usual: + +```json +{ + "summary": { + "total_runs": 5, + "total_duration": "2h30m", + "total_tokens": 45000, + "total_cost": 0.23 + }, + "runs": [...], + "tool_usage": [...], + ... +} +``` + +### Guardrail Triggered (Output > Token Limit) + +When the output exceeds the token limit (default: 12000 tokens), the command returns a structured response with: + +```json +{ + "message": "⚠️ Output size (15000 tokens) exceeds the limit (12000 tokens). To reduce output size, use the 'jq' parameter with one of the suggested queries below.", + "output_tokens": 15000, + "output_size_limit": 12000, + "schema": { + "description": "Complete structured data for workflow logs", + "type": "object", + "fields": { + "summary": { + "type": "object", + "description": "Aggregate metrics across all runs (total_runs, total_duration, total_tokens, total_cost, total_turns, total_errors, total_warnings, total_missing_tools)" + }, + "runs": { + "type": "array", + "description": "Array of workflow run data (database_id, workflow_name, agent, status, conclusion, duration, token_usage, estimated_cost, turns, error_count, warning_count, missing_tool_count, created_at, url, logs_path, event, branch)" + }, + ... + } + }, + "suggested_queries": [ + { + "description": "Get only the summary statistics", + "query": ".summary", + "example": "Use jq parameter: \".summary\"" + }, + { + "description": "Get list of run IDs and workflow names", + "query": ".runs | map({database_id, workflow_name, status})", + "example": "Use jq parameter: \".runs | map({database_id, workflow_name, status})\"" + }, + ... + ] +} +``` + +## Configuring the Token Limit + +The guardrail uses a token-based limit instead of byte-based. By default, the limit is 12000 tokens (approximately 48KB of text). + +You can customize the limit using the `max_tokens` parameter: + +```json +{ + "name": "logs", + "arguments": { + "count": 100, + "max_tokens": 20000 + } +} +``` + +**Token Estimation**: The system uses approximately 4 characters per token as an estimation (OpenAI's rule of thumb). + +## Using the jq Parameter + +The `jq` parameter allows you to filter the output using jq syntax. Here are the suggested queries: + +### 1. Get Only Summary Statistics + +```json +{ + "jq": ".summary" +} +``` + +Returns just the aggregate metrics without individual run data. + +### 2. Get Run IDs and Basic Info + +```json +{ + "jq": ".runs | map({database_id, workflow_name, status})" +} +``` + +Returns a simplified list of runs with just the essential fields. + +### 3. Get Only Failed Runs + +```json +{ + "jq": ".runs | map(select(.conclusion == \"failure\"))" +} +``` + +Filters to show only runs that failed. + +### 4. Get Summary with First N Runs + +```json +{ + "jq": "{summary, runs: .runs[:5]}" +} +``` + +Returns summary plus the first 5 runs only. + +### 5. Get Error and Warning Summaries + +```json +{ + "jq": "{errors_and_warnings, missing_tools, mcp_failures}" +} +``` + +Returns only the diagnostic information. + +### 6. Get Tool Usage Statistics + +```json +{ + "jq": ".tool_usage" +} +``` + +Returns aggregated tool usage data. + +### 7. Get High Token Usage Runs + +```json +{ + "jq": ".runs | map(select(.token_usage > 10000))" +} +``` + +Filters to show only runs with high token usage. + +### 8. Get Runs from Specific Workflow + +```json +{ + "jq": ".runs | map(select(.workflow_name == \"YOUR_WORKFLOW_NAME\"))" +} +``` + +Filters to show runs from a specific workflow. + +## Implementation Details + +### Constants + +- `DefaultMaxMCPLogsOutputTokens`: 12000 tokens (default limit) +- `CharsPerToken`: 4 characters per token (estimation factor) + +### Files + +- `pkg/cli/mcp_logs_guardrail.go` - Core guardrail implementation +- `pkg/cli/mcp_logs_guardrail_test.go` - Unit tests +- `pkg/cli/mcp_logs_guardrail_integration_test.go` - Integration tests +- `pkg/cli/mcp_server.go` - Integration with MCP server + +### Functions + +- `estimateTokens(text string) int` - Estimates token count from text +- `checkLogsOutputSize(outputStr string, maxTokens int) (string, bool)` - Main guardrail function +- `getLogsDataSchema() LogsDataSchema` - Returns schema description +- `getSuggestedJqQueries() []SuggestedJqQuery` - Returns suggested jq filters + +### Testing + +Run the tests with: + +```bash +# Unit tests +go test -v ./pkg/cli/mcp_logs_guardrail_test.go ./pkg/cli/mcp_logs_guardrail.go ./pkg/cli/jq.go + +# Integration tests +go test -v -tags integration -run "TestMCPServer_LogsGuardrail" ./pkg/cli/ + +# All tests +make test-unit +``` + +## Benefits + +1. **Prevents overwhelming responses** - Keeps output manageable for AI models +2. **Provides guidance** - Suggests specific filters to get the data you need +3. **Self-documenting** - Returns the schema so you know what fields are available +4. **Preserves functionality** - jq filtering works the same as before +5. **Transparent** - Clear messaging about why guardrail triggered + +## Future Enhancements + +Potential improvements: + +- Make the size limit configurable via parameter +- Add more sophisticated query suggestions based on output content +- Provide automatic chunking for very large datasets +- Add content-aware compression (e.g., gzip) for JSON responses to reduce transfer size while maintaining full data access