diff --git a/pkg/modelsdev/README.md b/pkg/modelsdev/README.md index a009b647d79..2f167c8c6a5 100644 --- a/pkg/modelsdev/README.md +++ b/pkg/modelsdev/README.md @@ -1,12 +1,10 @@ # modelsdev Package -The `modelsdev` package provides model pricing lookup backed by the public `models.dev` catalog. +The `modelsdev` package provides provider and model identifier normalization helpers. ## Overview -This package downloads and parses `https://models.dev/catalog.json`, normalizes provider/model identifiers, and exposes per-token pricing for callers that need cost-aware behavior. - -The catalog is loaded once per process via a singleton cache (`syncutil.OnceLoader`) to avoid repeated network fetches. +This package normalizes provider/model identifiers so callers can perform consistent model matching. ## Public API @@ -14,7 +12,6 @@ The catalog is loaded once per process via a singleton cache (`syncutil.OnceLoad | Function | Signature | Description | |----------|-----------|-------------| -| `FindPricing` | `func(ctx context.Context, provider, model string) (map[string]float64, bool)` | Returns normalized per-token pricing for a provider/model pair. Falls back to cross-provider matching when provider lookup fails. Returns `(nil, false)` when no pricing is available | | `NormalizeProvider` | `func(provider string) string` | Normalizes provider aliases such as `github`, `copilot`, and `github_models` to `github-copilot`, and lower-cases other provider identifiers | | `NormalizeComparableModelID` | `func(value string) string` | Lower-cases a model identifier, trims surrounding whitespace, and replaces `.` and `_` with `-` so equivalent model IDs compare consistently | @@ -23,31 +20,20 @@ The catalog is loaded once per process via a singleton cache (`syncutil.OnceLoad ```go import "github.com/github/gh-aw/pkg/modelsdev" -pricing, ok := modelsdev.FindPricing(ctx, "github", "gpt-4.1") -if !ok { - // pricing unavailable - return -} - -inputUSD := pricing["input"] // per token -outputUSD := pricing["output"] // per token -_ = inputUSD -_ = outputUSD +provider := modelsdev.NormalizeProvider(" github_models ") +comparableModel := modelsdev.NormalizeComparableModelID(" GPT_4.1-mini ") +_, _ = provider, comparableModel ``` ## Dependencies **Internal**: -- `github.com/github/gh-aw/pkg/logger` — debug logging -- `github.com/github/gh-aw/pkg/syncutil` — one-time catalog load/cache primitive +- None ## Design Notes - Provider aliases such as `github`, `copilot`, and `github_models` are normalized to `github-copilot`. - Comparable model matching normalizes separators (`.` and `_` to `-`) to improve lookup robustness. -- Numeric catalog costs are interpreted as per-million-token values and converted to per-token units. -- String catalog costs are treated as already normalized per-token values. -- Network or parsing failures degrade gracefully to an empty cache so callers can continue without pricing data. ## Source Synchronization diff --git a/pkg/modelsdev/catalog.go b/pkg/modelsdev/catalog.go index b13cec27ee9..f029857b9ce 100644 --- a/pkg/modelsdev/catalog.go +++ b/pkg/modelsdev/catalog.go @@ -1,222 +1,11 @@ package modelsdev -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strconv" - "strings" - "time" - - "github.com/github/gh-aw/pkg/logger" - "github.com/github/gh-aw/pkg/syncutil" -) - -const ( - fetchTimeout = 5 * time.Second - maxBodyBytes = 4 * 1024 * 1024 // 4 MiB safety cap -) - -// catalogURL is a variable so tests can override it with a local HTTP server. -var catalogURL = "https://models.dev/catalog.json" +import "strings" // modelIDReplacer normalizes separator characters in model IDs so that IDs // differing only in ".", "_", or "-" compare equal. var modelIDReplacer = strings.NewReplacer(".", "-", "_", "-") -var pkgLog = logger.New("modelsdev:catalog") - -// rawCatalog mirrors the top-level models.dev catalog JSON structure. -type rawCatalog struct { - Providers map[string]rawProvider `json:"providers"` -} - -type rawProvider struct { - Models map[string]rawModel `json:"models"` -} - -type rawModel struct { - // Cost values are per-million-token numbers (or pre-normalized strings) in the catalog. - Cost map[string]json.RawMessage `json:"cost"` -} - -// pricingCache maps normalizedProvider → normalizedModel → per-token pricing. -type pricingCache = map[string]map[string]map[string]float64 - -var ( - catalogCache syncutil.OnceLoader[pricingCache] - - // httpClientFactory is overridable for tests. - httpClientFactory = func() *http.Client { - return &http.Client{Timeout: fetchTimeout} - } -) - -// FindPricing looks up per-token pricing for the given provider/model from the downloaded -// models.dev catalog. Returns (nil, false) when the catalog is unavailable or the model -// is not found. -func FindPricing(ctx context.Context, provider, model string) (map[string]float64, bool) { - catalog := ensureCatalog(ctx) - if len(catalog) == 0 { - return nil, false - } - - normalizedProvider := NormalizeProvider(provider) - trimmedModel := strings.TrimSpace(model) - if trimmedModel == "" { - return nil, false - } - normalizedModel := strings.ToLower(trimmedModel) - comparableModel := NormalizeComparableModelID(normalizedModel) - - pkgLog.Printf("FindPricing: looking up provider=%q model=%q", normalizedProvider, normalizedModel) - - // Provider-scoped exact match. - if normalizedProvider != "" { - if providerModels, ok := catalog[normalizedProvider]; ok { - if pricing, ok := providerModels[normalizedModel]; ok { - pkgLog.Printf("FindPricing: provider-scoped exact match for %q/%q", normalizedProvider, normalizedModel) - return pricing, true - } - // Comparable (dot/underscore-normalized) model ID match. - for mn, pricing := range providerModels { - if NormalizeComparableModelID(mn) == comparableModel { - pkgLog.Printf("FindPricing: provider-scoped comparable match %q for %q", mn, normalizedModel) - return pricing, true - } - } - } - } - - // Cross-provider fallback (when provider is unknown or empty). - for _, providerModels := range catalog { - if pricing, ok := providerModels[normalizedModel]; ok { - pkgLog.Printf("FindPricing: cross-provider fallback match for model %q", normalizedModel) - return pricing, true - } - for mn, pricing := range providerModels { - if NormalizeComparableModelID(mn) == comparableModel { - pkgLog.Printf("FindPricing: cross-provider comparable match %q for %q", mn, normalizedModel) - return pricing, true - } - } - } - - pkgLog.Printf("FindPricing: no pricing found for provider=%q model=%q", normalizedProvider, normalizedModel) - return nil, false -} - -// ensureCatalog downloads and normalizes the models.dev pricing catalog at most once per -// process. Network failures are logged and result in an empty (non-nil) cache so -// subsequent calls are instant no-ops. -func ensureCatalog(ctx context.Context) pricingCache { - downloaded, _ := catalogCache.Get(func() (pricingCache, error) { - downloaded, err := downloadAndParseCatalog(ctx) - if err != nil { - pkgLog.Printf("models.dev catalog download failed (pricing fallback unavailable): %v", err) - return pricingCache{}, nil - } else { - total := 0 - for _, models := range downloaded { - total += len(models) - } - pkgLog.Printf("Downloaded models.dev catalog: %d providers, %d total models", len(downloaded), total) - } - return downloaded, nil - }) - return downloaded -} - -func downloadAndParseCatalog(ctx context.Context) (pricingCache, error) { - reqCtx, cancel := context.WithTimeout(ctx, fetchTimeout) - defer cancel() - - req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, catalogURL, nil) - if err != nil { - return nil, fmt.Errorf("creating request: %w", err) - } - - resp, err := httpClientFactory().Do(req) - if err != nil { - return nil, fmt.Errorf("GET %s: %w", catalogURL, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected HTTP %d from %s", resp.StatusCode, catalogURL) - } - - body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) - if err != nil { - return nil, fmt.Errorf("reading response: %w", err) - } - - return parseCatalog(body) -} - -// parseCatalog parses the raw models.dev catalog JSON and normalizes pricing to per-token -// float64 values. Numeric catalog values are in USD per-million tokens and are divided by -// 1,000,000; string values are treated as already per-token. -func parseCatalog(data []byte) (pricingCache, error) { - var raw rawCatalog - if err := json.Unmarshal(data, &raw); err != nil { - return nil, fmt.Errorf("parsing models.dev catalog JSON: %w", err) - } - - parsed := make(pricingCache) - for providerName, provider := range raw.Providers { - normalizedProvider := NormalizeProvider(providerName) - if normalizedProvider == "" { - continue - } - if parsed[normalizedProvider] == nil { - parsed[normalizedProvider] = make(map[string]map[string]float64) - } - for modelName, model := range provider.Models { - trimmedModel := strings.TrimSpace(modelName) - if trimmedModel == "" { - continue - } - normalizedModel := strings.ToLower(trimmedModel) - pricing := parseCostMap(model.Cost) - if len(pricing) > 0 { - parsed[normalizedProvider][normalizedModel] = pricing - } - } - } - return parsed, nil -} - -// parseCostMap converts a raw cost map from models.dev (per-million numbers or -// already-normalized per-token strings) into per-token float64 values. -func parseCostMap(raw map[string]json.RawMessage) map[string]float64 { - if len(raw) == 0 { - return nil - } - result := make(map[string]float64, len(raw)) - for key, val := range raw { - if len(val) == 0 { - continue - } - // Attempt numeric decode — models.dev stores prices per million tokens. - var f float64 - if err := json.Unmarshal(val, &f); err == nil { - result[key] = f / 1_000_000 // convert per-million → per-token - continue - } - // Fall back to string decode (pre-normalized per-token string values). - var s string - if err := json.Unmarshal(val, &s); err == nil { - if parsed, err := strconv.ParseFloat(strings.TrimSpace(s), 64); err == nil { - result[key] = parsed - } - } - } - return result -} - // NormalizeProvider maps provider aliases (e.g. "github", "copilot", "github_models") // to their canonical form ("github-copilot") and lower-cases all other values. func NormalizeProvider(provider string) string { diff --git a/pkg/modelsdev/catalog_test.go b/pkg/modelsdev/catalog_test.go deleted file mode 100644 index 63ed5a4b95a..00000000000 --- a/pkg/modelsdev/catalog_test.go +++ /dev/null @@ -1,181 +0,0 @@ -//go:build !integration - -package modelsdev - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// sampleCatalog is a minimal models.dev catalog JSON used in tests. -const sampleCatalog = `{ - "providers": { - "anthropic": { - "models": { - "claude-new-model": { - "cost": {"input": 3.0, "output": 15.0} - }, - "claude-no-cost": {} - } - }, - "openai": { - "models": { - "gpt-99": { - "cost": {"input": 2.5, "output": 10.0, "cache_read": 1.25} - } - } - }, - "unknown-provider": { - "models": { - "some-model": { - "cost": {"input": 1.0} - } - } - } - } -}` - -func TestParseCatalog(t *testing.T) { - parsed, err := parseCatalog([]byte(sampleCatalog)) - require.NoError(t, err) - - // Anthropic claude-new-model should be present with per-token pricing. - require.Contains(t, parsed, "anthropic") - require.Contains(t, parsed["anthropic"], "claude-new-model") - pricing := parsed["anthropic"]["claude-new-model"] - assert.InDelta(t, 3.0/1_000_000, pricing["input"], 1e-15) - assert.InDelta(t, 15.0/1_000_000, pricing["output"], 1e-15) - - // Models without cost should be excluded. - assert.NotContains(t, parsed["anthropic"], "claude-no-cost") - - // OpenAI gpt-99 should be present. - require.Contains(t, parsed, "openai") - require.Contains(t, parsed["openai"], "gpt-99") - oaiPricing := parsed["openai"]["gpt-99"] - assert.InDelta(t, 2.5/1_000_000, oaiPricing["input"], 1e-15) - assert.InDelta(t, 1.25/1_000_000, oaiPricing["cache_read"], 1e-15) - - // unknown-provider is lowercased and retained (normalizeProvider does not filter). - assert.Contains(t, parsed, "unknown-provider") -} - -func TestParseCostMap(t *testing.T) { - cases := []struct { - name string - raw map[string]json.RawMessage - want map[string]float64 - }{ - { - name: "numeric per-million values", - raw: map[string]json.RawMessage{ - "input": json.RawMessage("3.0"), - "output": json.RawMessage("15.0"), - }, - want: map[string]float64{"input": 3.0 / 1_000_000, "output": 15.0 / 1_000_000}, - }, - { - name: "string per-token values", - raw: map[string]json.RawMessage{ - "input": json.RawMessage(`"0.000003"`), - }, - want: map[string]float64{"input": 0.000003}, - }, - { - name: "empty map", - raw: nil, - want: nil, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := parseCostMap(tc.raw) - if tc.want == nil { - assert.Nil(t, got) - return - } - for k, v := range tc.want { - assert.InDeltaf(t, v, got[k], 1e-15, "key %q", k) - } - }) - } -} - -func TestFindPricing(t *testing.T) { - origURL := catalogURL - origFactory := httpClientFactory - t.Cleanup(func() { - catalogCache.Reset() - catalogURL = origURL - httpClientFactory = origFactory - }) - - catalogCache.Reset() - - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(sampleCatalog)) - })) - defer srv.Close() - - catalogURL = srv.URL - httpClientFactory = func() *http.Client { return srv.Client() } - - t.Run("found_exact_provider_and_model", func(t *testing.T) { - pricing, ok := FindPricing(context.Background(), "anthropic", "claude-new-model") - require.True(t, ok) - assert.InDelta(t, 3.0/1_000_000, pricing["input"], 1e-15) - }) - - t.Run("cross_provider_fallback", func(t *testing.T) { - pricing, ok := FindPricing(context.Background(), "", "gpt-99") - require.True(t, ok) - assert.Contains(t, pricing, "input") - }) - - t.Run("not_found_returns_false", func(t *testing.T) { - pricing, ok := FindPricing(context.Background(), "anthropic", "does-not-exist") - assert.False(t, ok) - assert.Nil(t, pricing) - }) -} - -func TestNormalizeProvider(t *testing.T) { - cases := []struct{ input, want string }{ - {"github", "github-copilot"}, - {"copilot", "github-copilot"}, - {"github_models", "github-copilot"}, - {"GITHUB_MODELS", "github-copilot"}, - {"anthropic", "anthropic"}, - {"OpenAI", "openai"}, - {" Anthropic ", "anthropic"}, - {"", ""}, - } - for _, tc := range cases { - t.Run(tc.input+"->"+tc.want, func(t *testing.T) { - assert.Equal(t, tc.want, NormalizeProvider(tc.input)) - }) - } -} - -func TestNormalizeComparableModelID(t *testing.T) { - cases := []struct{ input, want string }{ - {"claude-sonnet-4.6", "claude-sonnet-4-6"}, - {"gpt_4o", "gpt-4o"}, - {"GPT-4O", "gpt-4o"}, - {" claude.3 ", "claude-3"}, - {"", ""}, - } - for _, tc := range cases { - t.Run(tc.input+"->"+tc.want, func(t *testing.T) { - assert.Equal(t, tc.want, NormalizeComparableModelID(tc.input)) - }) - } -} diff --git a/pkg/modelsdev/spec_test.go b/pkg/modelsdev/spec_test.go index 4bd0856c0d2..1e1e8322a93 100644 --- a/pkg/modelsdev/spec_test.go +++ b/pkg/modelsdev/spec_test.go @@ -3,59 +3,11 @@ package modelsdev import ( - "context" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -// TestSpec_PublicAPI_FindPricing validates the documented behavior of -// FindPricing as described in the modelsdev README.md specification. -func TestSpec_PublicAPI_FindPricing(t *testing.T) { - ctx := context.Background() - - t.Run("returns nil false when pricing is unavailable", func(t *testing.T) { - pricing, ok := FindPricing(ctx, "definitely-not-a-provider", "definitely-not-a-model") - assert.False(t, ok, "FindPricing should report unavailable pricing for an unknown provider/model pair") - assert.Nil(t, pricing, "FindPricing should return nil pricing when no pricing is available") - }) - - t.Run("result pricing map exposes per token input and output entries when available", func(t *testing.T) { - pricing, ok := FindPricing(ctx, "github", "gpt-4.1") - if !ok { - t.Skip("catalog pricing unavailable in test environment; README specifies graceful degradation when network or parsing fails") - } - - require.NotNil(t, pricing, "FindPricing should return a non-nil pricing map when pricing is available") - _, hasInput := pricing["input"] - _, hasOutput := pricing["output"] - assert.True(t, hasInput, "pricing map should contain documented input per-token price entry") - assert.True(t, hasOutput, "pricing map should contain documented output per-token price entry") - }) -} - -// TestSpec_DesignDecision_ProviderAliases validates the documented provider -// alias normalization described in the modelsdev README.md. -func TestSpec_DesignDecision_ProviderAliases(t *testing.T) { - tests := []struct { - name string - provider string - }{ - {name: "github alias", provider: "github"}, - {name: "copilot alias", provider: "copilot"}, - {name: "github_models alias", provider: "github_models"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pricing, ok := FindPricing(context.Background(), tt.provider, "definitely-not-a-real-model") - assert.False(t, ok, "FindPricing should still return unavailable for an unknown model after normalizing provider alias %q", tt.provider) - assert.Nil(t, pricing, "FindPricing should return nil pricing for unknown model with provider alias %q", tt.provider) - }) - } -} - // TestSpec_PublicAPI_NormalizeProvider validates the documented alias and // case-normalization behavior of NormalizeProvider as described in the // modelsdev README.md specification.