From 232731bf448233c8c09a959eefdd854edaadd761 Mon Sep 17 00:00:00 2001 From: rainxchzed Date: Mon, 4 May 2026 16:57:30 +0500 Subject: [PATCH] Bump search bucket to 240/min and add user repos/starred passthrough --- CLAUDE.md | 2 + .../kotlin/zed/rainxch/githubstore/Plugins.kt | 18 +++-- .../zed/rainxch/githubstore/routes/Routing.kt | 2 + .../githubstore/routes/UserReposRoutes.kt | 70 ++++++++++++++++++ .../githubstore/routes/UserStarredRoutes.kt | 71 +++++++++++++++++++ 5 files changed, 158 insertions(+), 5 deletions(-) create mode 100644 src/main/kotlin/zed/rainxch/githubstore/routes/UserReposRoutes.kt create mode 100644 src/main/kotlin/zed/rainxch/githubstore/routes/UserStarredRoutes.kt diff --git a/CLAUDE.md b/CLAUDE.md index 6bf0515..5dc614b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,6 +54,8 @@ All under `/v1/`: | `GET /releases/{owner}/{name}?page=&per_page=` | Proxied list of GitHub releases. Reads optional `X-GitHub-Token`. Cached server-side for 1h. | | `GET /readme/{owner}/{name}` | Proxied README JSON (base64-encoded content + metadata, GitHub's shape). Reads optional `X-GitHub-Token`. Cached 24h. | | `GET /user/{username}` | Proxied GitHub user/org profile. Reads optional `X-GitHub-Token`. Cached 7d. | +| `GET /users/{username}/repos?type=&sort=&direction=&page=&per_page=` | Proxied list of a user/org's repos. `type` ∈ {all, owner, member}, `sort` ∈ {created, updated, pushed, full_name}, `direction` ∈ {asc, desc}. Whitelisted to block SSRF via query injection. Cached 1h server-side, edge `s-maxage=1800`. Reads `X-GitHub-Token`. | +| `GET /users/{username}/starred?sort=&direction=&page=&per_page=` | Proxied list of a user's starred repos (the public form -- the OAuth viewer-self form is intentionally NOT proxied). `sort` ∈ {created, updated}. Cached 30min server-side, edge `s-maxage=900`. Reads `X-GitHub-Token`. | | `POST /events` | Batched telemetry (opt-in, max 50 per batch). These rows drive `SignalAggregationWorker` — ranking only improves if clients send events. | | `GET /announcements` | Public, anonymous announcements feed. Same byte-identical envelope for every caller. Backed by JSON files in `src/main/resources/announcements/.json` (or `ANNOUNCEMENTS_DIR` env override). Validator enforces every rule from `docs/backend/announcements-endpoint.md` §2 at load time; expired items are filtered at serve time. `Cache-Control: public, max-age=600` + ETag revalidation. No auth, no per-user logic, no logging beyond standard access. | | `POST /auth/device/start` | Stateless proxy for `github.com/login/device/code`. Client used to call GitHub directly; some user networks (documented in OpenHub-Store/GitHub-Store#433, #395) can't reach GitHub reliably. Backend adds `client_id`, forwards GitHub's body verbatim. 10 req/hr/IP. | diff --git a/src/main/kotlin/zed/rainxch/githubstore/Plugins.kt b/src/main/kotlin/zed/rainxch/githubstore/Plugins.kt index 0cb4316..9e076b9 100644 --- a/src/main/kotlin/zed/rainxch/githubstore/Plugins.kt +++ b/src/main/kotlin/zed/rainxch/githubstore/Plugins.kt @@ -167,12 +167,20 @@ fun Application.configureHTTP() { rateLimiter(limit = 3, refillPeriod = 1.minutes) requestKey(::forwardedFor) } - // Search bucket: 60/min/key. Covers /search, /search/explore, - // /releases, /readme, /user — every route that fans out to the - // GitHub API. Keyed by token-hash when present, IP otherwise (see - // searchBucketKey for rationale). + // Search bucket: 240/min/key. Covers /search, /search/explore, + // /releases, /readme, /user, /users/{u}/repos, /users/{u}/starred -- + // every route that fans out to the GitHub API. Keyed by token-hash + // when present, IP otherwise (see searchBucketKey for rationale). + // + // Bumped from 60 -> 240 after observing real client burst patterns: + // a single details-page open can fan out to /repo + /releases + + // /readme + /user, and the developer-profile screen further pulls + // /users/{u}/repos + /users/{u}/starred. The aggregate 4-token pool + // (~20k/hr to GitHub) and per-route Cloudflare s-maxage absorb the + // real upstream load -- backend's own bucket was the constraint, not + // GitHub's quota. register(RateLimitName("search")) { - rateLimiter(limit = 60, refillPeriod = 1.minutes) + rateLimiter(limit = 240, refillPeriod = 1.minutes) requestKey(::searchBucketKey) } // Badges: 60/min/IP. Embedded in READMEs so a single popular repo can diff --git a/src/main/kotlin/zed/rainxch/githubstore/routes/Routing.kt b/src/main/kotlin/zed/rainxch/githubstore/routes/Routing.kt index a5fda47..2a3e031 100644 --- a/src/main/kotlin/zed/rainxch/githubstore/routes/Routing.kt +++ b/src/main/kotlin/zed/rainxch/githubstore/routes/Routing.kt @@ -51,6 +51,8 @@ fun Application.configureRouting() { releasesRoutes(resourceClient) readmeRoutes(resourceClient) userRoutes(resourceClient) + userReposRoutes(resourceClient) + userStarredRoutes(resourceClient) } authRoutes(deviceClient) internalRoutes(searchMetrics, workerSupervisor) diff --git a/src/main/kotlin/zed/rainxch/githubstore/routes/UserReposRoutes.kt b/src/main/kotlin/zed/rainxch/githubstore/routes/UserReposRoutes.kt new file mode 100644 index 0000000..7e2260e --- /dev/null +++ b/src/main/kotlin/zed/rainxch/githubstore/routes/UserReposRoutes.kt @@ -0,0 +1,70 @@ +package zed.rainxch.githubstore.routes + +import io.ktor.http.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* +import zed.rainxch.githubstore.ingest.GitHubResourceClient +import zed.rainxch.githubstore.util.GitHubIdentifiers + +// Whitelisted to keep query-string-injected SSRF off the upstream URL. +// GitHub silently ignores unknown values; we reject so a typo doesn't return +// the wrong-shaped data quietly. +private val VALID_TYPES = setOf("all", "owner", "member") +private val VALID_SORTS = setOf("created", "updated", "pushed", "full_name") +private val VALID_DIRECTIONS = setOf("asc", "desc") + +// GET /v1/users/{username}/repos -- public passthrough for a user/org's +// repo list. Powers the developer profile screen. Heavily paginated and +// per-(username,page,sort,type) cached because the underlying answer +// rarely changes within an hour. +fun Route.userReposRoutes(resourceClient: GitHubResourceClient) { + get("/users/{username}/repos") { + val username = GitHubIdentifiers.validOwner(call.parameters["username"]) + ?: return@get call.respond(HttpStatusCode.BadRequest, mapOf("error" to "invalid_owner")) + + val type = call.request.queryParameters["type"]?.takeIf { it in VALID_TYPES } ?: "owner" + val sort = call.request.queryParameters["sort"]?.takeIf { it in VALID_SORTS } ?: "updated" + val direction = call.request.queryParameters["direction"]?.takeIf { it in VALID_DIRECTIONS } ?: "desc" + val page = (call.request.queryParameters["page"]?.toIntOrNull() ?: 1).coerceIn(1, 50) + val perPage = (call.request.queryParameters["per_page"]?.toIntOrNull() ?: 30).coerceIn(1, 100) + + val userToken = call.request.headers["X-GitHub-Token"]?.takeIf { it.isNotBlank() } + + val cacheKey = "user-repos:$username?type=$type&sort=$sort&dir=$direction&page=$page&pp=$perPage" + val upstreamUrl = + "https://api.github.com/users/$username/repos" + + "?type=$type&sort=$sort&direction=$direction&per_page=$perPage&page=$page" + + // 1h TTL: balance between "user pushes a new repo, profile screen + // needs to reflect within reason" and "the profile reload button + // shouldn't smash GitHub". Aligns with /releases TTL. + val ttlSeconds = 3_600L + + val result = resourceClient.fetchCached( + cacheKey = cacheKey, + upstreamUrl = upstreamUrl, + userToken = userToken, + ttlSeconds = ttlSeconds, + ) + + when (result) { + is GitHubResourceClient.Result.Hit -> { + call.response.header(HttpHeaders.CacheControl, "public, max-age=300, s-maxage=1800") + call.respondText(result.body, ContentType.parse(result.contentType), HttpStatusCode.OK) + } + is GitHubResourceClient.Result.StaleFallback -> { + call.response.header(HttpHeaders.CacheControl, "no-store") + call.response.header("X-Cache-State", "stale-fallback") + call.respondText(result.body, ContentType.parse(result.contentType), HttpStatusCode.OK) + } + is GitHubResourceClient.Result.NegativeHit -> { + call.response.header(HttpHeaders.CacheControl, "public, max-age=60, s-maxage=300") + call.respond(HttpStatusCode.fromValue(result.status), mapOf("error" to "upstream_${result.status}")) + } + is GitHubResourceClient.Result.UpstreamError -> { + call.respond(HttpStatusCode.BadGateway, mapOf("error" to "github_unreachable")) + } + } + } +} diff --git a/src/main/kotlin/zed/rainxch/githubstore/routes/UserStarredRoutes.kt b/src/main/kotlin/zed/rainxch/githubstore/routes/UserStarredRoutes.kt new file mode 100644 index 0000000..0f99737 --- /dev/null +++ b/src/main/kotlin/zed/rainxch/githubstore/routes/UserStarredRoutes.kt @@ -0,0 +1,71 @@ +package zed.rainxch.githubstore.routes + +import io.ktor.http.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* +import zed.rainxch.githubstore.ingest.GitHubResourceClient +import zed.rainxch.githubstore.util.GitHubIdentifiers + +// Whitelist sort/direction the same way userReposRoutes does -- prevents +// query-string-injected SSRF and silent default-fallback bugs. +private val VALID_STARRED_SORTS = setOf("created", "updated") +private val VALID_STARRED_DIRECTIONS = setOf("asc", "desc") + +// GET /v1/users/{username}/starred -- public passthrough for a user's +// starred repos. Powers the starred-repos picker on the profile screen. +// +// Only the /users/{username}/starred form is supported here; the +// authenticated /user/starred form (current viewer's stars) is OAuth-bound +// and would require a separate flow that wires X-GitHub-Token through +// without caching cross-user. Out of scope for this endpoint -- callers +// who want the viewer's own stars must call GitHub directly. +fun Route.userStarredRoutes(resourceClient: GitHubResourceClient) { + get("/users/{username}/starred") { + val username = GitHubIdentifiers.validOwner(call.parameters["username"]) + ?: return@get call.respond(HttpStatusCode.BadRequest, mapOf("error" to "invalid_owner")) + + val sort = call.request.queryParameters["sort"]?.takeIf { it in VALID_STARRED_SORTS } ?: "created" + val direction = call.request.queryParameters["direction"]?.takeIf { it in VALID_STARRED_DIRECTIONS } ?: "desc" + val page = (call.request.queryParameters["page"]?.toIntOrNull() ?: 1).coerceIn(1, 50) + val perPage = (call.request.queryParameters["per_page"]?.toIntOrNull() ?: 30).coerceIn(1, 100) + + val userToken = call.request.headers["X-GitHub-Token"]?.takeIf { it.isNotBlank() } + + val cacheKey = "user-starred:$username?sort=$sort&dir=$direction&page=$page&pp=$perPage" + val upstreamUrl = + "https://api.github.com/users/$username/starred" + + "?sort=$sort&direction=$direction&per_page=$perPage&page=$page" + + // 30min TTL: stars churn faster than repos for active users (a star + // takes 1 click). Edge cache keeps the picker UI snappy without + // staleness becoming visible during a session. + val ttlSeconds = 1_800L + + val result = resourceClient.fetchCached( + cacheKey = cacheKey, + upstreamUrl = upstreamUrl, + userToken = userToken, + ttlSeconds = ttlSeconds, + ) + + when (result) { + is GitHubResourceClient.Result.Hit -> { + call.response.header(HttpHeaders.CacheControl, "public, max-age=180, s-maxage=900") + call.respondText(result.body, ContentType.parse(result.contentType), HttpStatusCode.OK) + } + is GitHubResourceClient.Result.StaleFallback -> { + call.response.header(HttpHeaders.CacheControl, "no-store") + call.response.header("X-Cache-State", "stale-fallback") + call.respondText(result.body, ContentType.parse(result.contentType), HttpStatusCode.OK) + } + is GitHubResourceClient.Result.NegativeHit -> { + call.response.header(HttpHeaders.CacheControl, "public, max-age=60, s-maxage=300") + call.respond(HttpStatusCode.fromValue(result.status), mapOf("error" to "upstream_${result.status}")) + } + is GitHubResourceClient.Result.UpstreamError -> { + call.respond(HttpStatusCode.BadGateway, mapOf("error" to "github_unreachable")) + } + } + } +}