Skip to content

Commit 57fe0e6

Browse files
grokifyclaude
andcommitted
feat(rubric): add IntegerScore type for 1-5 scale scoring
Add IntegerScore type with constants for the 5-level evaluation scale: - ScoreUnacceptable (1), ScoreMajorRevisions (2), ScoreAcceptable (3) - ScoreGood (4), ScoreExcellent (5) Includes helper methods: - String() for human-readable labels - ToCategorical() to convert to pass/partial/fail - IsValid() to check range validity - ParseIntegerScore() to parse from int This scale is preferred for LLM judges as research shows they are unreliable at finer granularity than 5 levels. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 3a0169f commit 57fe0e6

2 files changed

Lines changed: 216 additions & 0 deletions

File tree

rubric/score.go

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package rubric
2+
3+
// IntegerScore represents a 1-5 integer evaluation score.
4+
// This scale is preferred for LLM judges as research shows they are
5+
// unreliable at finer granularity than 5 levels.
6+
type IntegerScore int
7+
8+
const (
9+
// ScoreUnacceptable indicates the spec does not meet requirements.
10+
ScoreUnacceptable IntegerScore = 1
11+
12+
// ScoreMajorRevisions indicates significant work is needed.
13+
ScoreMajorRevisions IntegerScore = 2
14+
15+
// ScoreAcceptable indicates minimum requirements are met.
16+
ScoreAcceptable IntegerScore = 3
17+
18+
// ScoreGood indicates the spec meets expectations well.
19+
ScoreGood IntegerScore = 4
20+
21+
// ScoreExcellent indicates the spec exceeds expectations.
22+
ScoreExcellent IntegerScore = 5
23+
)
24+
25+
// String returns the human-readable label for the score.
26+
func (s IntegerScore) String() string {
27+
labels := []string{"", "Unacceptable", "Major Revisions", "Acceptable", "Good", "Excellent"}
28+
if s >= 1 && s <= 5 {
29+
return labels[s]
30+
}
31+
return "Unknown"
32+
}
33+
34+
// ToCategorical converts the integer score to a categorical ScoreValue.
35+
// 1-2 = fail, 3 = partial, 4-5 = pass
36+
func (s IntegerScore) ToCategorical() ScoreValue {
37+
switch {
38+
case s <= 2:
39+
return ScoreFail
40+
case s == 3:
41+
return ScorePartial
42+
default:
43+
return ScorePass
44+
}
45+
}
46+
47+
// IsValid returns true if the score is in the valid 1-5 range.
48+
func (s IntegerScore) IsValid() bool {
49+
return s >= 1 && s <= 5
50+
}
51+
52+
// IsPassing returns true if the score is considered passing (4 or higher).
53+
func (s IntegerScore) IsPassing() bool {
54+
return s >= ScoreGood
55+
}
56+
57+
// Icon returns the emoji icon for the score.
58+
func (s IntegerScore) Icon() string {
59+
switch s {
60+
case ScoreExcellent:
61+
return "🌟"
62+
case ScoreGood:
63+
return "🟢"
64+
case ScoreAcceptable:
65+
return "🟡"
66+
case ScoreMajorRevisions:
67+
return "🟠"
68+
case ScoreUnacceptable:
69+
return "🔴"
70+
default:
71+
return "⚪"
72+
}
73+
}
74+
75+
// ParseIntegerScore converts an integer to IntegerScore, clamping to valid range.
76+
func ParseIntegerScore(score int) IntegerScore {
77+
if score < 1 {
78+
return ScoreUnacceptable
79+
}
80+
if score > 5 {
81+
return ScoreExcellent
82+
}
83+
return IntegerScore(score)
84+
}
85+
86+
// AllIntegerScores returns all valid integer scores in descending order.
87+
func AllIntegerScores() []IntegerScore {
88+
return []IntegerScore{
89+
ScoreExcellent,
90+
ScoreGood,
91+
ScoreAcceptable,
92+
ScoreMajorRevisions,
93+
ScoreUnacceptable,
94+
}
95+
}

rubric/score_test.go

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package rubric
2+
3+
import "testing"
4+
5+
func TestIntegerScore_String(t *testing.T) {
6+
tests := []struct {
7+
score IntegerScore
8+
want string
9+
}{
10+
{ScoreUnacceptable, "Unacceptable"},
11+
{ScoreMajorRevisions, "Major Revisions"},
12+
{ScoreAcceptable, "Acceptable"},
13+
{ScoreGood, "Good"},
14+
{ScoreExcellent, "Excellent"},
15+
{IntegerScore(0), "Unknown"},
16+
{IntegerScore(6), "Unknown"},
17+
}
18+
19+
for _, tt := range tests {
20+
t.Run(tt.want, func(t *testing.T) {
21+
if got := tt.score.String(); got != tt.want {
22+
t.Errorf("IntegerScore(%d).String() = %q, want %q", tt.score, got, tt.want)
23+
}
24+
})
25+
}
26+
}
27+
28+
func TestIntegerScore_ToCategorical(t *testing.T) {
29+
tests := []struct {
30+
score IntegerScore
31+
want ScoreValue
32+
}{
33+
{ScoreUnacceptable, ScoreFail},
34+
{ScoreMajorRevisions, ScoreFail},
35+
{ScoreAcceptable, ScorePartial},
36+
{ScoreGood, ScorePass},
37+
{ScoreExcellent, ScorePass},
38+
}
39+
40+
for _, tt := range tests {
41+
t.Run(tt.score.String(), func(t *testing.T) {
42+
if got := tt.score.ToCategorical(); got != tt.want {
43+
t.Errorf("IntegerScore(%d).ToCategorical() = %s, want %s", tt.score, got, tt.want)
44+
}
45+
})
46+
}
47+
}
48+
49+
func TestIntegerScore_IsValid(t *testing.T) {
50+
tests := []struct {
51+
score IntegerScore
52+
want bool
53+
}{
54+
{IntegerScore(0), false},
55+
{ScoreUnacceptable, true},
56+
{ScoreMajorRevisions, true},
57+
{ScoreAcceptable, true},
58+
{ScoreGood, true},
59+
{ScoreExcellent, true},
60+
{IntegerScore(6), false},
61+
}
62+
63+
for _, tt := range tests {
64+
if got := tt.score.IsValid(); got != tt.want {
65+
t.Errorf("IntegerScore(%d).IsValid() = %v, want %v", tt.score, got, tt.want)
66+
}
67+
}
68+
}
69+
70+
func TestIntegerScore_IsPassing(t *testing.T) {
71+
tests := []struct {
72+
score IntegerScore
73+
want bool
74+
}{
75+
{ScoreUnacceptable, false},
76+
{ScoreMajorRevisions, false},
77+
{ScoreAcceptable, false},
78+
{ScoreGood, true},
79+
{ScoreExcellent, true},
80+
}
81+
82+
for _, tt := range tests {
83+
t.Run(tt.score.String(), func(t *testing.T) {
84+
if got := tt.score.IsPassing(); got != tt.want {
85+
t.Errorf("IntegerScore(%d).IsPassing() = %v, want %v", tt.score, got, tt.want)
86+
}
87+
})
88+
}
89+
}
90+
91+
func TestParseIntegerScore(t *testing.T) {
92+
tests := []struct {
93+
input int
94+
want IntegerScore
95+
}{
96+
{-1, ScoreUnacceptable},
97+
{0, ScoreUnacceptable},
98+
{1, ScoreUnacceptable},
99+
{2, ScoreMajorRevisions},
100+
{3, ScoreAcceptable},
101+
{4, ScoreGood},
102+
{5, ScoreExcellent},
103+
{6, ScoreExcellent},
104+
{100, ScoreExcellent},
105+
}
106+
107+
for _, tt := range tests {
108+
if got := ParseIntegerScore(tt.input); got != tt.want {
109+
t.Errorf("ParseIntegerScore(%d) = %d, want %d", tt.input, got, tt.want)
110+
}
111+
}
112+
}
113+
114+
func TestIntegerScore_Icon(t *testing.T) {
115+
// Just verify icons are non-empty
116+
for _, score := range AllIntegerScores() {
117+
if icon := score.Icon(); icon == "" {
118+
t.Errorf("IntegerScore(%d).Icon() should not be empty", score)
119+
}
120+
}
121+
}

0 commit comments

Comments
 (0)