Skip to content

Commit fae0a2b

Browse files
grokifyclaude
andcommitted
feat(rubric): enhance CategoryResult with IntScore and Confidence
Add v2 evaluation fields to CategoryResult: - IntScore: 1-5 integer score for LLM judge compatibility - Confidence: 0.0-1.0 confidence level for human review routing - ReasonCodes: standardized finding identifiers for automated repair Add helper methods: - SetIntScore() derives categorical score from integer - SetConfidence() with bounds validation - AddReasonCode()/AddReasonCodes() for code management - HasLowConfidence() for review routing decisions - NewCategoryResultWithIntScore() factory function Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 456bd01 commit fae0a2b

1 file changed

Lines changed: 65 additions & 0 deletions

File tree

rubric/category.go

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,23 @@ type CategoryResult struct {
4747
// This is the authoritative score for decision-making.
4848
Score ScoreValue `json:"score"`
4949

50+
// IntScore is the 1-5 integer score.
51+
// Preferred for LLM judges as they are unreliable at finer granularity.
52+
IntScore IntegerScore `json:"intScore,omitempty"`
53+
5054
// NumericScore is an optional numeric score (e.g., 1-5 Likert).
5155
// Used for human comparison, inter-rater reliability, and calibration.
5256
// The categorical Score takes precedence for pass/fail decisions.
5357
NumericScore *float64 `json:"numericScore,omitempty"`
5458

59+
// Confidence is the evaluator's confidence in this score (0.0-1.0).
60+
// Low confidence scores may be routed to human review.
61+
Confidence float64 `json:"confidence,omitempty"`
62+
63+
// ReasonCodes are standardized finding identifiers for this category.
64+
// Enable automated repair workflows.
65+
ReasonCodes []ReasonCode `json:"reasonCodes,omitempty"`
66+
5567
// Reasoning explains the score (chain-of-thought).
5668
Reasoning string `json:"reasoning"`
5769

@@ -161,6 +173,59 @@ func (cr *CategoryResult) IsPassing() bool {
161173
return cr.Score.IsPassing()
162174
}
163175

176+
// SetIntScore sets the integer score and derives the categorical score.
177+
func (cr *CategoryResult) SetIntScore(score IntegerScore) *CategoryResult {
178+
cr.IntScore = score
179+
cr.Score = score.ToCategorical()
180+
return cr
181+
}
182+
183+
// SetConfidence sets the confidence value.
184+
func (cr *CategoryResult) SetConfidence(confidence float64) *CategoryResult {
185+
if confidence < 0 {
186+
confidence = 0
187+
}
188+
if confidence > 1 {
189+
confidence = 1
190+
}
191+
cr.Confidence = confidence
192+
return cr
193+
}
194+
195+
// AddReasonCode adds a reason code to this category result.
196+
func (cr *CategoryResult) AddReasonCode(code ReasonCode) *CategoryResult {
197+
cr.ReasonCodes = append(cr.ReasonCodes, code)
198+
return cr
199+
}
200+
201+
// AddReasonCodes adds multiple reason codes to this category result.
202+
func (cr *CategoryResult) AddReasonCodes(codes ...ReasonCode) *CategoryResult {
203+
cr.ReasonCodes = append(cr.ReasonCodes, codes...)
204+
return cr
205+
}
206+
207+
// HasLowConfidence returns true if confidence is below the threshold (default 0.7).
208+
func (cr *CategoryResult) HasLowConfidence(threshold ...float64) bool {
209+
t := 0.7
210+
if len(threshold) > 0 {
211+
t = threshold[0]
212+
}
213+
return cr.Confidence > 0 && cr.Confidence < t
214+
}
215+
216+
// NewCategoryResultWithIntScore creates a category result from an integer score.
217+
func NewCategoryResultWithIntScore(category string, intScore IntegerScore, confidence float64, reasoning string) *CategoryResult {
218+
return &CategoryResult{
219+
Category: category,
220+
IntScore: intScore,
221+
Score: intScore.ToCategorical(),
222+
Confidence: confidence,
223+
Reasoning: reasoning,
224+
Evidence: []string{},
225+
Findings: []Finding{},
226+
}
227+
}
228+
164229
// CountCategoryResults counts results by score value.
165230
type CategoryResultCounts struct {
166231
Pass int `json:"pass"`

0 commit comments

Comments
 (0)