diff --git a/services/apps/packages_worker/src/security-contacts/extractors/registry/go.ts b/services/apps/packages_worker/src/security-contacts/extractors/registry/go.ts new file mode 100644 index 0000000000..27145b09a4 --- /dev/null +++ b/services/apps/packages_worker/src/security-contacts/extractors/registry/go.ts @@ -0,0 +1,28 @@ +import { ExtractorResult, ProvenanceEntry, RawContact } from '../../types' + +const SOURCE = 'go.dev/security-policy' +const POLICY_URL = 'https://go.dev/doc/security/policy#reporting-a-security-bug' + +export async function fetchGo(): Promise { + const fetchedAt = new Date().toISOString() + const prov = (): ProvenanceEntry[] => [ + { source: SOURCE, sourceTier: 'D', path: POLICY_URL, fetchedAt }, + ] + const contacts: RawContact[] = [ + { + channel: 'email', + value: 'security@golang.org', + role: 'security-team', + tier: 'D', + provenance: prov(), + }, + { + channel: 'web-form', + value: 'https://g.co/vulnz', + role: 'security-team', + tier: 'D', + provenance: prov(), + }, + ] + return { contacts, policies: {} } +} diff --git a/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts b/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts index a539f711ed..dcd61f4c9f 100644 --- a/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts +++ b/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts @@ -2,6 +2,7 @@ import { Extractor, ExtractorResult, RawContact, RepoPolicies } from '../../type import { fetchCargo } from './cargo' import { fetchComposer } from './composer' +import { fetchGo } from './go' import { fetchMaven } from './maven' import { fetchNpm } from './npm' import { fetchNuget } from './nuget' @@ -16,7 +17,7 @@ type EcosystemFetcher = ( repoUrl?: string, ) => Promise -// Keyed by the lowercased packages.ecosystem value. go has no package-manifest contacts. +// Keyed by the lowercased packages.ecosystem value. const FETCHERS: Record = { npm: fetchNpm, pypi: fetchPypi, @@ -25,6 +26,7 @@ const FETCHERS: Record = { nuget: fetchNuget, rubygems: fetchRubygems, composer: fetchComposer, + go: fetchGo, } export const extractManifest: Extractor = async (target, deps) => { diff --git a/services/apps/packages_worker/src/security-contacts/extractors/repoOwner.ts b/services/apps/packages_worker/src/security-contacts/extractors/repoOwner.ts new file mode 100644 index 0000000000..ea9c106743 --- /dev/null +++ b/services/apps/packages_worker/src/security-contacts/extractors/repoOwner.ts @@ -0,0 +1,66 @@ +import { getServiceChildLogger } from '@crowd/logging' + +import { parseGithubUrl } from '../../enricher/fetchLightRepo' +import { ExtractorDeps, ProvenanceEntry, RawContact, RepoTarget } from '../types' + +import { isEmail } from './http' + +const log = getServiceChildLogger('security-contacts:repo-owner') + +const SOURCE = 'github-repo-owner' + +export function buildOwnerHandleContact(owner: string, fetchedAt: string, url: string): RawContact { + return { + channel: 'github-handle', + value: owner, + handle: owner, + role: 'org-owner', + tier: 'D', + provenance: [{ source: SOURCE, sourceTier: 'D', path: url, fetchedAt }], + } +} + +export async function fetchRepoOwner( + target: RepoTarget, + deps: Pick, +): Promise { + let owner: string + try { + ;({ owner } = parseGithubUrl(target.url)) + } catch { + return [] + } + + const fetchedAt = new Date().toISOString() + const contacts: RawContact[] = [buildOwnerHandleContact(owner, fetchedAt, target.url)] + + try { + const { text } = await deps.githubGet(`/users/${owner}`) + const email = (text ? (JSON.parse(text) as { email?: unknown }) : null)?.email + if (typeof email === 'string' && isEmail(email)) { + const provenance: ProvenanceEntry[] = [ + { + source: SOURCE, + sourceTier: 'D', + path: `https://api.github.com/users/${owner}`, + fetchedAt, + }, + ] + contacts.push({ + channel: 'email', + value: email, + handle: owner, + role: 'org-owner', + tier: 'D', + provenance, + }) + } + } catch (err) { + log.warn( + { repoId: target.repoId, owner, errMsg: (err as Error).message }, + 'Owner email lookup failed', + ) + } + + return contacts +} diff --git a/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts b/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts new file mode 100644 index 0000000000..89ee0ff053 --- /dev/null +++ b/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts @@ -0,0 +1,204 @@ +import { getServiceChildLogger } from '@crowd/logging' + +import { parseGithubUrl } from '../../enricher/fetchLightRepo' +import { ExtractorDeps, ProvenanceEntry, RawContact, RepoTarget } from '../types' + +import { isEmail } from './http' + +const log = getServiceChildLogger('security-contacts:top-committers') + +const SOURCE = 'github-commits' +const SINCE_DAYS = 90 +const TOP_N = 3 +// GitHub computes /stats/contributors asynchronously and returns 202 while it's not ready yet — +// poll a few times with a short backoff before giving up for this pass. +const STATS_POLL_ATTEMPTS = 3 +const STATS_POLL_DELAY_MS = 2000 + +interface WeekStat { + w?: unknown + c?: unknown +} + +interface ContributorStat { + author?: { login?: unknown; type?: unknown } | null + weeks?: WeekStat[] +} + +interface AggregatedAuthor { + login: string + count: number +} + +const BOT_TOKENS = [ + 'dependabot', + 'renovate', + 'github-actions', + 'claude', + 'anthropic', + 'copilot', + 'chatgpt', + 'openai', +] + +function matchesBotToken(value: string): boolean { + const lower = value.toLowerCase() + return BOT_TOKENS.some((token) => lower.includes(token)) +} + +function isBotLogin(login: string): boolean { + const lower = login.toLowerCase() + if (lower.endsWith('[bot]')) return true + return matchesBotToken(lower) +} + +export function aggregateTopCommitters( + stats: ContributorStat[], + sinceUnixSeconds: number, + topN: number = TOP_N, +): AggregatedAuthor[] { + const authors: AggregatedAuthor[] = [] + + for (const stat of stats) { + if (stat.author?.type === 'Bot') continue + const login = typeof stat.author?.login === 'string' ? stat.author.login : undefined + if (!login || isBotLogin(login)) continue + + const count = (stat.weeks ?? []).reduce((sum, week) => { + const w = typeof week.w === 'number' ? week.w : 0 + const c = typeof week.c === 'number' ? week.c : 0 + return w >= sinceUnixSeconds ? sum + c : sum + }, 0) + if (count > 0) authors.push({ login, count }) + } + + return authors.sort((a, b) => b.count - a.count || a.login.localeCompare(b.login)).slice(0, topN) +} + +async function resolvePublicEmail( + login: string, + githubGet: ExtractorDeps['githubGet'], +): Promise { + try { + const { text } = await githubGet(`/users/${login}`) + const email = (text ? (JSON.parse(text) as { email?: unknown }) : null)?.email + if (typeof email !== 'string' || !isEmail(email) || matchesBotToken(email)) return null + return email + } catch (err) { + log.warn({ login, errMsg: (err as Error).message }, 'Committer email resolution failed') + return null + } +} + +export async function mapCommittersToContacts( + authors: AggregatedAuthor[], + fetchedAt: string, + apiPath: string, + githubGet: ExtractorDeps['githubGet'], +): Promise { + const contacts: RawContact[] = [] + + for (const author of authors) { + const provenance: ProvenanceEntry[] = [ + { source: SOURCE, sourceTier: 'D', path: apiPath, fetchedAt }, + ] + + contacts.push({ + channel: 'github-handle', + value: author.login, + handle: author.login, + role: 'committer', + tier: 'D', + provenance, + }) + + const email = await resolvePublicEmail(author.login, githubGet) + if (email) { + contacts.push({ + channel: 'email', + value: email, + handle: author.login, + role: 'committer', + tier: 'D', + provenance, + }) + } + } + + return contacts +} + +async function fetchContributorStats( + path: string, + target: RepoTarget, + owner: string, + name: string, + githubGet: ExtractorDeps['githubGet'], + sleep: (ms: number) => Promise, +): Promise { + for (let attempt = 1; attempt <= STATS_POLL_ATTEMPTS; attempt++) { + let status: number + let text: string | null + try { + ;({ status, text } = await githubGet(path, { extraOkStatuses: [202] })) + } catch (err) { + log.warn( + { repoId: target.repoId, owner, name, errMsg: (err as Error).message }, + 'Contributor stats lookup failed', + ) + return null + } + + if (status === 202) { + if (attempt < STATS_POLL_ATTEMPTS) await sleep(STATS_POLL_DELAY_MS) + continue + } + + if (!text) return null + try { + const parsed = JSON.parse(text) + return Array.isArray(parsed) ? (parsed as ContributorStat[]) : null + } catch (err) { + log.warn( + { repoId: target.repoId, owner, name, errMsg: (err as Error).message }, + 'Contributor stats lookup failed', + ) + return null + } + } + + log.info( + { repoId: target.repoId, owner, name }, + 'Contributor stats still computing after polling — skipping this pass', + ) + return null +} + +export async function fetchTopCommitters( + target: RepoTarget, + deps: Pick, + now: () => Date = () => new Date(), + sleep: (ms: number) => Promise = (ms) => new Promise((r) => setTimeout(r, ms)), +): Promise { + let owner: string + let name: string + try { + ;({ owner, name } = parseGithubUrl(target.url)) + } catch { + return [] + } + + const path = `/repos/${owner}/${name}/stats/contributors` + const stats = await fetchContributorStats(path, target, owner, name, deps.githubGet, sleep) + if (!stats) return [] + + const sinceUnixSeconds = Math.floor(now().getTime() / 1000) - SINCE_DAYS * 24 * 60 * 60 + const authors = aggregateTopCommitters(stats, sinceUnixSeconds) + const fetchedAt = new Date().toISOString() + return mapCommittersToContacts( + authors, + fetchedAt, + `https://api.github.com${path}`, + deps.githubGet, + ) +} diff --git a/services/apps/packages_worker/src/security-contacts/githubToken.ts b/services/apps/packages_worker/src/security-contacts/githubToken.ts index c118269e96..393de6b53f 100644 --- a/services/apps/packages_worker/src/security-contacts/githubToken.ts +++ b/services/apps/packages_worker/src/security-contacts/githubToken.ts @@ -129,16 +129,19 @@ async function fetchOnce( export async function githubApiGet( path: string, timeoutMs: number, - opts: { raw?: boolean } = {}, + opts: { raw?: boolean; extraOkStatuses?: number[] } = {}, ): Promise { const accept = opts.raw ? 'application/vnd.github.raw' : 'application/vnd.github+json' + const okStatuses = opts.extraOkStatuses + ? new Set([...ABSENT_STATUSES, ...opts.extraOkStatuses]) + : ABSENT_STATUSES const url = `${GITHUB_API}${path}` const resolved = await ensurePool() if (!resolved) { const res = await fetchOnce(url, timeoutMs, { Accept: accept }) if (res.status === 200) return { status: 200, text: await res.text() } - if (ABSENT_STATUSES.has(res.status)) return { status: res.status, text: null } + if (okStatuses.has(res.status)) return { status: res.status, text: null } throw new Error(`githubApiGet ${path} failed: HTTP ${res.status}`) } @@ -166,7 +169,7 @@ export async function githubApiGet( Accept: accept, }) - if (res.status === 200 || ABSENT_STATUSES.has(res.status)) { + if (res.status === 200 || okStatuses.has(res.status)) { pool.parkIfBudgetLow( installationId, numOrNull(res.headers.get('x-ratelimit-remaining')), diff --git a/services/apps/packages_worker/src/security-contacts/processBatch.ts b/services/apps/packages_worker/src/security-contacts/processBatch.ts index bbde5eba71..c89832107c 100644 --- a/services/apps/packages_worker/src/security-contacts/processBatch.ts +++ b/services/apps/packages_worker/src/security-contacts/processBatch.ts @@ -1,5 +1,6 @@ import { cancellationSignal, heartbeat } from '@temporalio/activity' +import { classifyEmailReachability } from '@crowd/common' import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { getServiceChildLogger } from '@crowd/logging' @@ -10,12 +11,14 @@ import { mapWithConcurrency } from '../utils/concurrency' import { fetchRepoTree } from './extractors/gitTree' import { extractPvr } from './extractors/pvr' import { extractManifest } from './extractors/registry' +import { fetchRepoOwner } from './extractors/repoOwner' import { extractSecurityContactsFile } from './extractors/securityContactsFile' import { extractSecurityInsights } from './extractors/securityInsights' import { extractSecurityMd } from './extractors/securityMd' import { extractSecurityTxt } from './extractors/securityTxt' +import { fetchTopCommitters } from './extractors/topCommitters' import { githubApiGet } from './githubToken' -import { reconcile } from './reconcile' +import { isJunkContact, reconcile } from './reconcile' import { deriveGithubHandlesFromNoreplyEmails, resolveCdpEmails } from './resolveCdpEmails' import { Extractor, @@ -169,9 +172,21 @@ export async function processRepo( } contacts.push(...(await verifyHandleCandidates(target, deps, handleCandidates))) - contacts.push(...deriveGithubHandlesFromNoreplyEmails(contacts)) + const isUsableRaw = (c: RawContact): boolean => { + if (isJunkContact(c)) return false + return c.channel !== 'email' || classifyEmailReachability(c.value).reachable + } + const hasUsableHigherTierContact = contacts.some((c) => c.tier !== 'D' && isUsableRaw(c)) + if (!hasUsableHigherTierContact) { + const [committers, ownerContacts] = await Promise.all([ + fetchTopCommitters(target, deps), + fetchRepoOwner(target, deps), + ]) + contacts.push(...committers, ...ownerContacts) + } + const handleContacts = contacts.filter((c) => c.channel === 'github-handle') if (handleContacts.length > 0) { try { diff --git a/services/apps/packages_worker/src/security-contacts/reconcile.ts b/services/apps/packages_worker/src/security-contacts/reconcile.ts index 43206e59fc..edb3e73d77 100644 --- a/services/apps/packages_worker/src/security-contacts/reconcile.ts +++ b/services/apps/packages_worker/src/security-contacts/reconcile.ts @@ -45,7 +45,7 @@ function urlHost(value: string): string | null { } } -function isJunkContact(c: RawContact): boolean { +export function isJunkContact(c: RawContact): boolean { if (c.channel === 'email') { const domain = c.value.split('@')[1]?.toLowerCase().trim() return domain != null && PLACEHOLDER_EMAIL_DOMAINS.has(domain) diff --git a/services/apps/packages_worker/src/security-contacts/types.ts b/services/apps/packages_worker/src/security-contacts/types.ts index af84a0ea6d..26f2fbe79b 100644 --- a/services/apps/packages_worker/src/security-contacts/types.ts +++ b/services/apps/packages_worker/src/security-contacts/types.ts @@ -67,7 +67,7 @@ export interface RepoTarget { export interface GithubGetResult { status: number - /** Null for absent resources (404/410/422/451). */ + /** Null for absent resources (404/410/422/451) or a status listed in extraOkStatuses. */ text: string | null } @@ -75,7 +75,12 @@ export interface ExtractorDeps { fetchTimeoutMs: number /** Required — crates.io rejects requests without an identifying UA. */ userAgent: string - githubGet: (path: string, opts?: { raw?: boolean }) => Promise + /** extraOkStatuses: additional statuses to return as {status, text: null} instead of throwing — + * e.g. 202 for endpoints (like /stats/contributors) that mean "still computing", not absent. */ + githubGet: ( + path: string, + opts?: { raw?: boolean; extraOkStatuses?: number[] }, + ) => Promise /** Default-branch file paths from one git-tree call; null means unresolved — probe as before. */ repoTree: { paths: Set | null } }