Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 56 additions & 2 deletions pkg/cli/logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ Examples:
` + constants.CLIExtensionPrefix + ` logs -c 10 # Download last 10 runs
` + constants.CLIExtensionPrefix + ` logs --start-date 2024-01-01 # Filter runs after date
` + constants.CLIExtensionPrefix + ` logs --end-date 2024-01-31 # Filter runs before date
` + constants.CLIExtensionPrefix + ` logs --engine claude # Filter logs by claude engine
` + constants.CLIExtensionPrefix + ` logs --engine codex # Filter logs by codex engine
` + constants.CLIExtensionPrefix + ` logs -o ./my-logs # Custom output directory`,
Run: func(cmd *cobra.Command, args []string) {
var workflowName string
Expand Down Expand Up @@ -121,9 +123,23 @@ Examples:
startDate, _ := cmd.Flags().GetString("start-date")
endDate, _ := cmd.Flags().GetString("end-date")
outputDir, _ := cmd.Flags().GetString("output")
engine, _ := cmd.Flags().GetString("engine")
verbose, _ := cmd.Flags().GetBool("verbose")

if err := DownloadWorkflowLogs(workflowName, count, startDate, endDate, outputDir, verbose); err != nil {
// Validate engine parameter using the engine registry
if engine != "" {
registry := workflow.GetGlobalEngineRegistry()
if !registry.IsValidEngine(engine) {
supportedEngines := registry.GetSupportedEngines()
fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{
Type: "error",
Message: fmt.Sprintf("invalid engine value '%s'. Must be one of: %s", engine, strings.Join(supportedEngines, ", ")),
}))
os.Exit(1)
}
}

if err := DownloadWorkflowLogs(workflowName, count, startDate, endDate, outputDir, engine, verbose); err != nil {
fmt.Fprintln(os.Stderr, console.FormatError(console.CompilerError{
Type: "error",
Message: err.Error(),
Expand All @@ -138,12 +154,13 @@ Examples:
logsCmd.Flags().String("start-date", "", "Filter runs created after this date (YYYY-MM-DD)")
logsCmd.Flags().String("end-date", "", "Filter runs created before this date (YYYY-MM-DD)")
logsCmd.Flags().StringP("output", "o", "./logs", "Output directory for downloaded logs and artifacts")
logsCmd.Flags().String("engine", "", "Filter logs by agentic engine type (claude, codex)")

return logsCmd
}

// DownloadWorkflowLogs downloads and analyzes workflow logs with metrics
func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, outputDir string, verbose bool) error {
func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, outputDir, engine string, verbose bool) error {
if verbose {
fmt.Println(console.FormatInfoMessage("Fetching workflow runs from GitHub Actions..."))
}
Expand Down Expand Up @@ -221,6 +238,43 @@ func DownloadWorkflowLogs(workflowName string, count int, startDate, endDate, ou
continue
}

// Apply engine filtering if specified
if engine != "" {
// Check if the run's engine matches the filter
awInfoPath := filepath.Join(result.LogsPath, "aw_info.json")
detectedEngine := extractEngineFromAwInfo(awInfoPath, verbose)

var engineMatches bool
if detectedEngine != nil {
// Get the engine ID to compare with the filter
registry := workflow.GetGlobalEngineRegistry()
for _, supportedEngine := range []string{"claude", "codex"} {
if testEngine, err := registry.GetEngine(supportedEngine); err == nil && testEngine == detectedEngine {
engineMatches = (supportedEngine == engine)
break
}
}
}

if !engineMatches {
if verbose {
engineName := "unknown"
if detectedEngine != nil {
// Try to get a readable name for the detected engine
registry := workflow.GetGlobalEngineRegistry()
for _, supportedEngine := range []string{"claude", "codex"} {
if testEngine, err := registry.GetEngine(supportedEngine); err == nil && testEngine == detectedEngine {
engineName = supportedEngine
break
}
}
}
fmt.Println(console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: engine '%s' does not match filter '%s'", result.Run.DatabaseID, engineName, engine)))
}
continue
}
}

// Update run with metrics and path
run := result.Run
run.TokenUsage = result.Metrics.TokenUsage
Expand Down
86 changes: 85 additions & 1 deletion pkg/cli/logs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func TestDownloadWorkflowLogs(t *testing.T) {
// Test the DownloadWorkflowLogs function
// This should either fail with auth error (if not authenticated)
// or succeed with no results (if authenticated but no workflows match)
err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", false)
err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", "", false)

// If GitHub CLI is authenticated, the function may succeed but find no results
// If not authenticated, it should return an auth error
Expand Down Expand Up @@ -733,3 +733,87 @@ input_tokens: 2000`
t.Errorf("Expected token usage 0 (no aw_info.json), got %d", metrics.TokenUsage)
}
}

func TestDownloadWorkflowLogsWithEngineFilter(t *testing.T) {
// Test that the engine filter parameter is properly validated and passed through
tests := []struct {
name string
engine string
expectError bool
errorText string
}{
{
name: "valid claude engine",
engine: "claude",
expectError: false,
},
{
name: "valid codex engine",
engine: "codex",
expectError: false,
},
{
name: "empty engine (no filter)",
engine: "",
expectError: false,
},
{
name: "invalid engine",
engine: "gpt",
expectError: true,
errorText: "invalid engine value 'gpt'",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// This function should validate the engine parameter
// If invalid, it would exit in the actual command but we can't test that easily
// So we just test that valid engines don't cause immediate errors
if !tt.expectError {
// For valid engines, test that the function can be called without panic
// It may still fail with auth errors, which is expected
err := DownloadWorkflowLogs("", 1, "", "", "./test-logs", tt.engine, false)

// Clean up any created directories
os.RemoveAll("./test-logs")

// If there's an error, it should be auth or workflow-related, not parameter validation
if err != nil {
errMsg := strings.ToLower(err.Error())
if strings.Contains(errMsg, "invalid engine") {
t.Errorf("Got engine validation error for valid engine '%s': %v", tt.engine, err)
}
}
}
})
}
}
func TestLogsCommandFlags(t *testing.T) {
// Test that the logs command has the expected flags including the new engine flag
cmd := NewLogsCommand()

// Check that all expected flags are present
expectedFlags := []string{"count", "start-date", "end-date", "output", "engine"}

for _, flagName := range expectedFlags {
flag := cmd.Flags().Lookup(flagName)
if flag == nil {
t.Errorf("Expected flag '%s' not found in logs command", flagName)
}
}

// Test engine flag specifically
engineFlag := cmd.Flags().Lookup("engine")
if engineFlag == nil {
t.Fatal("Engine flag not found")
}

if engineFlag.Usage != "Filter logs by agentic engine type (claude, codex)" {
t.Errorf("Unexpected engine flag usage text: %s", engineFlag.Usage)
}

if engineFlag.DefValue != "" {
t.Errorf("Expected engine flag default value to be empty, got: %s", engineFlag.DefValue)
}
}
26 changes: 26 additions & 0 deletions pkg/cli/templates/instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,30 @@ tools:
Respond to @helper-bot mentions with helpful information.
```

## Workflow Monitoring and Analysis

### Logs and Metrics

Monitor workflow execution and costs using the `logs` command:

```bash
# Download logs for all agentic workflows
gh aw logs

# Download logs for a specific workflow
gh aw logs weekly-research

# Filter logs by AI engine type
gh aw logs --engine claude # Only Claude workflows
gh aw logs --engine codex # Only Codex workflows

# Limit number of runs and filter by date
gh aw logs -c 10 --start-date 2024-01-01 --end-date 2024-01-31

# Download to custom directory
gh aw logs -o ./workflow-logs
```

## Security Considerations

### Cross-Prompt Injection Protection
Expand Down Expand Up @@ -486,6 +510,8 @@ Agentic workflows compile to GitHub Actions YAML:
7. **Set `stop-time`** for cost-sensitive workflows
8. **Set `max-turns`** to limit chat iterations and prevent runaway loops
9. **Use specific tool permissions** rather than broad access
10. **Monitor costs with `gh aw logs`** to track AI model usage and expenses
11. **Use `--engine` filter** in logs command to analyze specific AI engine performance

## Validation

Expand Down