diff --git a/pkg/cli/logs.go b/pkg/cli/logs.go index aebe744cc3e..a81da5a4839 100644 --- a/pkg/cli/logs.go +++ b/pkg/cli/logs.go @@ -90,6 +90,8 @@ Examples: ` + constants.CLIExtensionPrefix + ` logs -c 10 # Download last 10 runs ` + constants.CLIExtensionPrefix + ` logs --start-date 2024-01-01 # Filter runs after date ` + constants.CLIExtensionPrefix + ` logs --end-date 2024-01-31 # Filter runs before date + ` + constants.CLIExtensionPrefix + ` logs --engine claude # Filter logs by claude engine + ` + constants.CLIExtensionPrefix + ` logs --engine codex # Filter logs by codex engine ` + constants.CLIExtensionPrefix + ` logs -o ./my-logs # Custom output directory`, Run: func(cmd *cobra.Command, args []string) { var workflowName string @@ -121,9 +123,23 @@ Examples: startDate, _ := cmd.Flags().GetString("start-date") endDate, _ := cmd.Flags().GetString("end-date") outputDir, _ := cmd.Flags().GetString("output") + engine, _ := cmd.Flags().GetString("engine") verbose, _ := cmd.Flags().GetBool("verbose") - if err := DownloadWorkflowLogs(workflowName, count, startDate, endDate, outputDir, verbose); err != nil { + // Validate engine parameter using the engine registry + if engine != "" { + registry := workflow.GetGlobalEngineRegistry() + if !registry.IsValidEngine(engine) { + supportedEngines := registry.GetSupportedEngines() + fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{ + Type: "error", + Message: fmt.Sprintf("invalid engine value '%s'. Must be one of: %s", engine, strings.Join(supportedEngines, ", ")), + })) + os.Exit(1) + } + } + + if err := DownloadWorkflowLogs(workflowName, count, startDate, endDate, outputDir, engine, verbose); err != nil { fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{ Type: "error", Message: err.Error(), @@ -138,12 +154,13 @@ Examples: logsCmd.Flags().String("start-date", "", "Filter runs created after this date (YYYY-MM-DD)") logsCmd.Flags().String("end-date", "", "Filter runs created before this date (YYYY-MM-DD)") logsCmd.Flags().StringP("output", "o", "./logs", "Output directory for downloaded logs and artifacts") + logsCmd.Flags().String("engine", "", "Filter logs by agentic engine type (claude, codex)") return logsCmd } // DownloadWorkflowLogs downloads and analyzes workflow logs with metrics -func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, outputDir string, verbose bool) error { +func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, outputDir, engine string, verbose bool) error { if verbose { fmt.Println(console.FormatInfoMessage("Fetching workflow runs from GitHub Actions...")) } @@ -221,6 +238,43 @@ func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, ou continue } + // Apply engine filtering if specified + if engine != "" { + // Check if the run's engine matches the filter + awInfoPath := filepath.Join(result.LogsPath, "aw_info.json") + detectedEngine := extractEngineFromAwInfo(awInfoPath, verbose) + + var engineMatches bool + if detectedEngine != nil { + // Get the engine ID to compare with the filter + registry := workflow.GetGlobalEngineRegistry() + for _, supportedEngine := range []string{"claude", "codex"} { + if testEngine, err := registry.GetEngine(supportedEngine); err == nil && testEngine == detectedEngine { + engineMatches = (supportedEngine == engine) + break + } + } + } + + if !engineMatches { + if verbose { + engineName := "unknown" + if detectedEngine != nil { + // Try to get a readable name for the detected engine + registry := workflow.GetGlobalEngineRegistry() + for _, supportedEngine := range []string{"claude", "codex"} { + if testEngine, err := registry.GetEngine(supportedEngine); err == nil && testEngine == detectedEngine { + engineName = supportedEngine + break + } + } + } + fmt.Println(console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: engine '%s' does not match filter '%s'", result.Run.DatabaseID, engineName, engine))) + } + continue + } + } + // Update run with metrics and path run := result.Run run.TokenUsage = result.Metrics.TokenUsage diff --git a/pkg/cli/logs_test.go b/pkg/cli/logs_test.go index 9dd3e77d0ad..79f4810b430 100644 --- a/pkg/cli/logs_test.go +++ b/pkg/cli/logs_test.go @@ -15,7 +15,7 @@ func TestDownloadWorkflowLogs(t *testing.T) { // Test the DownloadWorkflowLogs function // This should either fail with auth error (if not authenticated) // or succeed with no results (if authenticated but no workflows match) - err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", false) + err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", "", false) // If GitHub CLI is authenticated, the function may succeed but find no results // If not authenticated, it should return an auth error @@ -733,3 +733,87 @@ input_tokens: 2000` t.Errorf("Expected token usage 0 (no aw_info.json), got %d", metrics.TokenUsage) } } + +func TestDownloadWorkflowLogsWithEngineFilter(t *testing.T) { + // Test that the engine filter parameter is properly validated and passed through + tests := []struct { + name string + engine string + expectError bool + errorText string + }{ + { + name: "valid claude engine", + engine: "claude", + expectError: false, + }, + { + name: "valid codex engine", + engine: "codex", + expectError: false, + }, + { + name: "empty engine (no filter)", + engine: "", + expectError: false, + }, + { + name: "invalid engine", + engine: "gpt", + expectError: true, + errorText: "invalid engine value 'gpt'", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // This function should validate the engine parameter + // If invalid, it would exit in the actual command but we can't test that easily + // So we just test that valid engines don't cause immediate errors + if !tt.expectError { + // For valid engines, test that the function can be called without panic + // It may still fail with auth errors, which is expected + err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", tt.engine, false) + + // Clean up any created directories + os.RemoveAll("./test-logs") + + // If there's an error, it should be auth or workflow-related, not parameter validation + if err != nil { + errMsg := strings.ToLower(err.Error()) + if strings.Contains(errMsg, "invalid engine") { + t.Errorf("Got engine validation error for valid engine '%s': %v", tt.engine, err) + } + } + } + }) + } +} +func TestLogsCommandFlags(t *testing.T) { + // Test that the logs command has the expected flags including the new engine flag + cmd := NewLogsCommand() + + // Check that all expected flags are present + expectedFlags := []string{"count", "start-date", "end-date", "output", "engine"} + + for _, flagName := range expectedFlags { + flag := cmd.Flags().Lookup(flagName) + if flag == nil { + t.Errorf("Expected flag '%s' not found in logs command", flagName) + } + } + + // Test engine flag specifically + engineFlag := cmd.Flags().Lookup("engine") + if engineFlag == nil { + t.Fatal("Engine flag not found") + } + + if engineFlag.Usage != "Filter logs by agentic engine type (claude, codex)" { + t.Errorf("Unexpected engine flag usage text: %s", engineFlag.Usage) + } + + if engineFlag.DefValue != "" { + t.Errorf("Expected engine flag default value to be empty, got: %s", engineFlag.DefValue) + } +} diff --git a/pkg/cli/templates/instructions.md b/pkg/cli/templates/instructions.md index 6982a5e6c58..1a860dc0653 100644 --- a/pkg/cli/templates/instructions.md +++ b/pkg/cli/templates/instructions.md @@ -407,6 +407,30 @@ tools: Respond to @helper-bot mentions with helpful information. ``` +## Workflow Monitoring and Analysis + +### Logs and Metrics + +Monitor workflow execution and costs using the `logs` command: + +```bash +# Download logs for all agentic workflows +gh aw logs + +# Download logs for a specific workflow +gh aw logs weekly-research + +# Filter logs by AI engine type +gh aw logs --engine claude # Only Claude workflows +gh aw logs --engine codex # Only Codex workflows + +# Limit number of runs and filter by date +gh aw logs -c 10 --start-date 2024-01-01 --end-date 2024-01-31 + +# Download to custom directory +gh aw logs -o ./workflow-logs +``` + ## Security Considerations ### Cross-Prompt Injection Protection @@ -486,6 +510,8 @@ Agentic workflows compile to GitHub Actions YAML: 7. **Set `stop-time`** for cost-sensitive workflows 8. **Set `max-turns`** to limit chat iterations and prevent runaway loops 9. **Use specific tool permissions** rather than broad access +10. **Monitor costs with `gh aw logs`** to track AI model usage and expenses +11. **Use `--engine` filter** in logs command to analyze specific AI engine performance ## Validation