Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ All under `/v1/`:
| `GET /releases/{owner}/{name}?page=&per_page=` | Proxied list of GitHub releases. Reads optional `X-GitHub-Token`. Cached server-side for 1h. |
| `GET /readme/{owner}/{name}` | Proxied README JSON (base64-encoded content + metadata, GitHub's shape). Reads optional `X-GitHub-Token`. Cached 24h. |
| `GET /user/{username}` | Proxied GitHub user/org profile. Reads optional `X-GitHub-Token`. Cached 7d. |
| `GET /users/{username}/repos?type=&sort=&direction=&page=&per_page=` | Proxied list of a user/org's repos. `type` ∈ {all, owner, member}, `sort` ∈ {created, updated, pushed, full_name}, `direction` ∈ {asc, desc}. Whitelisted to block SSRF via query injection. Cached 1h server-side, edge `s-maxage=1800`. Reads `X-GitHub-Token`. |
| `GET /users/{username}/starred?sort=&direction=&page=&per_page=` | Proxied list of a user's starred repos (the public form -- the OAuth viewer-self form is intentionally NOT proxied). `sort` ∈ {created, updated}. Cached 30min server-side, edge `s-maxage=900`. Reads `X-GitHub-Token`. |
| `POST /events` | Batched telemetry (opt-in, max 50 per batch). These rows drive `SignalAggregationWorker` — ranking only improves if clients send events. |
| `GET /announcements` | Public, anonymous announcements feed. Same byte-identical envelope for every caller. Backed by JSON files in `src/main/resources/announcements/<id>.json` (or `ANNOUNCEMENTS_DIR` env override). Validator enforces every rule from `docs/backend/announcements-endpoint.md` §2 at load time; expired items are filtered at serve time. `Cache-Control: public, max-age=600` + ETag revalidation. No auth, no per-user logic, no logging beyond standard access. |
| `POST /auth/device/start` | Stateless proxy for `github.com/login/device/code`. Client used to call GitHub directly; some user networks (documented in OpenHub-Store/GitHub-Store#433, #395) can't reach GitHub reliably. Backend adds `client_id`, forwards GitHub's body verbatim. 10 req/hr/IP. |
Expand Down
18 changes: 13 additions & 5 deletions src/main/kotlin/zed/rainxch/githubstore/Plugins.kt
Original file line number Diff line number Diff line change
Expand Up @@ -167,12 +167,20 @@ fun Application.configureHTTP() {
rateLimiter(limit = 3, refillPeriod = 1.minutes)
requestKey(::forwardedFor)
}
// Search bucket: 60/min/key. Covers /search, /search/explore,
// /releases, /readme, /user — every route that fans out to the
// GitHub API. Keyed by token-hash when present, IP otherwise (see
// searchBucketKey for rationale).
// Search bucket: 240/min/key. Covers /search, /search/explore,
// /releases, /readme, /user, /users/{u}/repos, /users/{u}/starred --
// every route that fans out to the GitHub API. Keyed by token-hash
// when present, IP otherwise (see searchBucketKey for rationale).
//
// Bumped from 60 -> 240 after observing real client burst patterns:
// a single details-page open can fan out to /repo + /releases +
// /readme + /user, and the developer-profile screen further pulls
// /users/{u}/repos + /users/{u}/starred. The aggregate 4-token pool
// (~20k/hr to GitHub) and per-route Cloudflare s-maxage absorb the
// real upstream load -- backend's own bucket was the constraint, not
// GitHub's quota.
register(RateLimitName("search")) {
rateLimiter(limit = 60, refillPeriod = 1.minutes)
rateLimiter(limit = 240, refillPeriod = 1.minutes)
requestKey(::searchBucketKey)
}
// Badges: 60/min/IP. Embedded in READMEs so a single popular repo can
Expand Down
2 changes: 2 additions & 0 deletions src/main/kotlin/zed/rainxch/githubstore/routes/Routing.kt
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ fun Application.configureRouting() {
releasesRoutes(resourceClient)
readmeRoutes(resourceClient)
userRoutes(resourceClient)
userReposRoutes(resourceClient)
userStarredRoutes(resourceClient)
}
authRoutes(deviceClient)
internalRoutes(searchMetrics, workerSupervisor)
Expand Down
70 changes: 70 additions & 0 deletions src/main/kotlin/zed/rainxch/githubstore/routes/UserReposRoutes.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package zed.rainxch.githubstore.routes

import io.ktor.http.*
import io.ktor.server.request.*
import io.ktor.server.response.*
import io.ktor.server.routing.*
import zed.rainxch.githubstore.ingest.GitHubResourceClient
import zed.rainxch.githubstore.util.GitHubIdentifiers

// Whitelisted to keep query-string-injected SSRF off the upstream URL.
// GitHub silently ignores unknown values; we reject so a typo doesn't return
// the wrong-shaped data quietly.
private val VALID_TYPES = setOf("all", "owner", "member")
private val VALID_SORTS = setOf("created", "updated", "pushed", "full_name")
private val VALID_DIRECTIONS = setOf("asc", "desc")

// GET /v1/users/{username}/repos -- public passthrough for a user/org's
// repo list. Powers the developer profile screen. Heavily paginated and
// per-(username,page,sort,type) cached because the underlying answer
// rarely changes within an hour.
fun Route.userReposRoutes(resourceClient: GitHubResourceClient) {
get("/users/{username}/repos") {
val username = GitHubIdentifiers.validOwner(call.parameters["username"])
?: return@get call.respond(HttpStatusCode.BadRequest, mapOf("error" to "invalid_owner"))

val type = call.request.queryParameters["type"]?.takeIf { it in VALID_TYPES } ?: "owner"
val sort = call.request.queryParameters["sort"]?.takeIf { it in VALID_SORTS } ?: "updated"
val direction = call.request.queryParameters["direction"]?.takeIf { it in VALID_DIRECTIONS } ?: "desc"
val page = (call.request.queryParameters["page"]?.toIntOrNull() ?: 1).coerceIn(1, 50)
val perPage = (call.request.queryParameters["per_page"]?.toIntOrNull() ?: 30).coerceIn(1, 100)

val userToken = call.request.headers["X-GitHub-Token"]?.takeIf { it.isNotBlank() }

val cacheKey = "user-repos:$username?type=$type&sort=$sort&dir=$direction&page=$page&pp=$perPage"
val upstreamUrl =
"https://api.github.com/users/$username/repos" +
"?type=$type&sort=$sort&direction=$direction&per_page=$perPage&page=$page"

// 1h TTL: balance between "user pushes a new repo, profile screen
// needs to reflect within reason" and "the profile reload button
// shouldn't smash GitHub". Aligns with /releases TTL.
val ttlSeconds = 3_600L

val result = resourceClient.fetchCached(
cacheKey = cacheKey,
upstreamUrl = upstreamUrl,
userToken = userToken,
ttlSeconds = ttlSeconds,
)

when (result) {
is GitHubResourceClient.Result.Hit -> {
call.response.header(HttpHeaders.CacheControl, "public, max-age=300, s-maxage=1800")
call.respondText(result.body, ContentType.parse(result.contentType), HttpStatusCode.OK)
}
is GitHubResourceClient.Result.StaleFallback -> {
call.response.header(HttpHeaders.CacheControl, "no-store")
call.response.header("X-Cache-State", "stale-fallback")
call.respondText(result.body, ContentType.parse(result.contentType), HttpStatusCode.OK)
}
is GitHubResourceClient.Result.NegativeHit -> {
call.response.header(HttpHeaders.CacheControl, "public, max-age=60, s-maxage=300")
call.respond(HttpStatusCode.fromValue(result.status), mapOf("error" to "upstream_${result.status}"))
}
is GitHubResourceClient.Result.UpstreamError -> {
call.respond(HttpStatusCode.BadGateway, mapOf("error" to "github_unreachable"))
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package zed.rainxch.githubstore.routes

import io.ktor.http.*
import io.ktor.server.request.*
import io.ktor.server.response.*
import io.ktor.server.routing.*
import zed.rainxch.githubstore.ingest.GitHubResourceClient
import zed.rainxch.githubstore.util.GitHubIdentifiers

// Whitelist sort/direction the same way userReposRoutes does -- prevents
// query-string-injected SSRF and silent default-fallback bugs.
private val VALID_STARRED_SORTS = setOf("created", "updated")
private val VALID_STARRED_DIRECTIONS = setOf("asc", "desc")

// GET /v1/users/{username}/starred -- public passthrough for a user's
// starred repos. Powers the starred-repos picker on the profile screen.
//
// Only the /users/{username}/starred form is supported here; the
// authenticated /user/starred form (current viewer's stars) is OAuth-bound
// and would require a separate flow that wires X-GitHub-Token through
// without caching cross-user. Out of scope for this endpoint -- callers
// who want the viewer's own stars must call GitHub directly.
fun Route.userStarredRoutes(resourceClient: GitHubResourceClient) {
get("/users/{username}/starred") {
val username = GitHubIdentifiers.validOwner(call.parameters["username"])
?: return@get call.respond(HttpStatusCode.BadRequest, mapOf("error" to "invalid_owner"))

val sort = call.request.queryParameters["sort"]?.takeIf { it in VALID_STARRED_SORTS } ?: "created"
val direction = call.request.queryParameters["direction"]?.takeIf { it in VALID_STARRED_DIRECTIONS } ?: "desc"
val page = (call.request.queryParameters["page"]?.toIntOrNull() ?: 1).coerceIn(1, 50)
val perPage = (call.request.queryParameters["per_page"]?.toIntOrNull() ?: 30).coerceIn(1, 100)

val userToken = call.request.headers["X-GitHub-Token"]?.takeIf { it.isNotBlank() }

val cacheKey = "user-starred:$username?sort=$sort&dir=$direction&page=$page&pp=$perPage"
val upstreamUrl =
"https://api.github.com/users/$username/starred" +
"?sort=$sort&direction=$direction&per_page=$perPage&page=$page"

// 30min TTL: stars churn faster than repos for active users (a star
// takes 1 click). Edge cache keeps the picker UI snappy without
// staleness becoming visible during a session.
val ttlSeconds = 1_800L

val result = resourceClient.fetchCached(
cacheKey = cacheKey,
upstreamUrl = upstreamUrl,
userToken = userToken,
ttlSeconds = ttlSeconds,
)

when (result) {
is GitHubResourceClient.Result.Hit -> {
call.response.header(HttpHeaders.CacheControl, "public, max-age=180, s-maxage=900")
call.respondText(result.body, ContentType.parse(result.contentType), HttpStatusCode.OK)
}
is GitHubResourceClient.Result.StaleFallback -> {
call.response.header(HttpHeaders.CacheControl, "no-store")
call.response.header("X-Cache-State", "stale-fallback")
call.respondText(result.body, ContentType.parse(result.contentType), HttpStatusCode.OK)
}
is GitHubResourceClient.Result.NegativeHit -> {
call.response.header(HttpHeaders.CacheControl, "public, max-age=60, s-maxage=300")
call.respond(HttpStatusCode.fromValue(result.status), mapOf("error" to "upstream_${result.status}"))
}
is GitHubResourceClient.Result.UpstreamError -> {
call.respond(HttpStatusCode.BadGateway, mapOf("error" to "github_unreachable"))
}
}
}
}
Loading