Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 81 additions & 3 deletions pkg/cli/audit_expanded.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,11 @@ type AuditEngineConfig struct {

// PromptAnalysis represents analysis of the input prompt
type PromptAnalysis struct {
PromptSize int `json:"prompt_size" console:"header:Prompt Size (chars)"`
PromptFile string `json:"prompt_file,omitempty" console:"header:Prompt File,omitempty"`
PromptSize int `json:"prompt_size" console:"header:Prompt Size (chars)"`
PromptFile string `json:"prompt_file,omitempty" console:"header:Prompt File,omitempty"`
CodeBlocksByLang map[string]int `json:"code_blocks_by_language,omitempty"`
JavaScriptPrograms int `json:"javascript_programs,omitempty" console:"header:JavaScript Programs,omitempty"`
JavaScriptTotalSize int `json:"javascript_total_size,omitempty" console:"header:JavaScript Total Size (chars),omitempty"`
}

// SessionAnalysis represents session and agent performance metrics
Expand Down Expand Up @@ -281,15 +284,90 @@ func extractPromptAnalysis(logsPath string) *PromptAnalysis {
PromptSize: len(data),
PromptFile: relPromptPath,
}
populatePromptCodeBlockAnalysis(string(data), analysis)

auditExpandedLog.Printf("Extracted prompt analysis: size=%d chars from %s", analysis.PromptSize, relPromptPath)
auditExpandedLog.Printf("Extracted prompt analysis: size=%d chars, javascript_programs=%d from %s", analysis.PromptSize, analysis.JavaScriptPrograms, relPromptPath)
return analysis
}

auditExpandedLog.Printf("No prompt.txt found in %s", logsPath)
return nil
}

func populatePromptCodeBlockAnalysis(content string, analysis *PromptAnalysis) {
lines := strings.Split(content, "\n")
analysis.CodeBlocksByLang = make(map[string]int)

inCodeBlock := false
fenceChar := byte(0)
fenceLen := 0
currentLang := ""
currentSize := 0

for _, line := range lines {
trimmed := strings.TrimSpace(line)
char, length, lang, isFence := parseMarkdownFence(trimmed)

if !inCodeBlock {
if !isFence {
continue
}
inCodeBlock = true
fenceChar = char
fenceLen = length
currentLang = lang
currentSize = 0
continue
}

if isFence && char == fenceChar && length >= fenceLen {
if currentLang != "" {
analysis.CodeBlocksByLang[currentLang]++
if currentLang == "js" || currentLang == "javascript" {
analysis.JavaScriptPrograms++
analysis.JavaScriptTotalSize += currentSize
}
}
inCodeBlock = false
fenceChar = 0
fenceLen = 0
currentLang = ""
currentSize = 0
continue
}

currentSize += len(line)
}

if len(analysis.CodeBlocksByLang) == 0 {
analysis.CodeBlocksByLang = nil
}
}

func parseMarkdownFence(line string) (char byte, length int, lang string, ok bool) {
if len(line) < 3 {
return 0, 0, "", false
}
first := line[0]
if first != '`' && first != '~' {
return 0, 0, "", false
}

i := 0
for i < len(line) && line[i] == first {
i++
}
if i < 3 {
return 0, 0, "", false
}

lang = strings.TrimSpace(line[i:])
if lang != "" {
lang = strings.ToLower(strings.Fields(lang)[0])
}
return first, i, lang, true
}

// buildSessionAnalysis creates session performance metrics from available data
func buildSessionAnalysis(processedRun ProcessedRun, metrics LogMetrics) *SessionAnalysis {
run := processedRun.Run
Expand Down
16 changes: 16 additions & 0 deletions pkg/cli/audit_expanded_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ func TestExtractPromptAnalysis(t *testing.T) {
expectNil bool
expectedSize int
expectedRelPath string // expected relative path in PromptFile
expectedJSCount int
expectedJSSize int
}{
{
name: "prompt in root directory",
Expand All @@ -150,6 +152,17 @@ func TestExtractPromptAnalysis(t *testing.T) {
expectedSize: 38,
expectedRelPath: "prompt.txt",
},
{
name: "prompt with javascript fenced blocks",
promptContent: "Intro\n```javascript\nconst a = 1;\nconst b = 2;\n```\n" +
"Middle\n```js\nconsole.log('ok')\n```\n" +
"```python\nprint('x')\n```",
promptDir: "",
expectedSize: 109,
expectedRelPath: "prompt.txt",
expectedJSCount: 2,
expectedJSSize: 41,
},
{
name: "prompt in aw-prompts subdirectory",
promptContent: "Another test prompt.",
Expand Down Expand Up @@ -211,6 +224,8 @@ func TestExtractPromptAnalysis(t *testing.T) {
require.NotNil(t, result, "Prompt analysis should not be nil")
assert.Equal(t, tt.expectedSize, result.PromptSize, "Prompt size should match")
assert.Equal(t, tt.expectedRelPath, result.PromptFile, "Prompt file should be a relative path")
assert.Equal(t, tt.expectedJSCount, result.JavaScriptPrograms, "JavaScript fenced block count should match")
assert.Equal(t, tt.expectedJSSize, result.JavaScriptTotalSize, "JavaScript fenced block size should match")
})
}
}
Expand Down Expand Up @@ -614,6 +629,7 @@ func TestBuildAuditDataWithExpandedSections(t *testing.T) {
require.NotNil(t, auditData.PromptAnalysis, "Prompt analysis should be populated")
assert.Len(t, promptContent, auditData.PromptAnalysis.PromptSize, "Prompt size should match")
assert.Equal(t, filepath.Join("activation", "aw-prompts", "prompt.txt"), auditData.PromptAnalysis.PromptFile, "Prompt file should be a relative path")
assert.Equal(t, 0, auditData.PromptAnalysis.JavaScriptPrograms, "JavaScript program count should default to 0 when none exist")
})

t.Run("SessionAnalysis", func(t *testing.T) {
Expand Down