From 5a5bcd8b3d7af0c7cf0f649dc8807aba8267d741 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 21 Jul 2026 11:18:22 +0100 Subject: [PATCH 1/5] feat: add golang.org security contact fallback for go packages Signed-off-by: Mouad BANI --- .../extractors/registry/go.ts | 28 +++++++++++++++++++ .../extractors/registry/index.ts | 4 ++- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 services/apps/packages_worker/src/security-contacts/extractors/registry/go.ts diff --git a/services/apps/packages_worker/src/security-contacts/extractors/registry/go.ts b/services/apps/packages_worker/src/security-contacts/extractors/registry/go.ts new file mode 100644 index 0000000000..27145b09a4 --- /dev/null +++ b/services/apps/packages_worker/src/security-contacts/extractors/registry/go.ts @@ -0,0 +1,28 @@ +import { ExtractorResult, ProvenanceEntry, RawContact } from '../../types' + +const SOURCE = 'go.dev/security-policy' +const POLICY_URL = 'https://go.dev/doc/security/policy#reporting-a-security-bug' + +export async function fetchGo(): Promise { + const fetchedAt = new Date().toISOString() + const prov = (): ProvenanceEntry[] => [ + { source: SOURCE, sourceTier: 'D', path: POLICY_URL, fetchedAt }, + ] + const contacts: RawContact[] = [ + { + channel: 'email', + value: 'security@golang.org', + role: 'security-team', + tier: 'D', + provenance: prov(), + }, + { + channel: 'web-form', + value: 'https://g.co/vulnz', + role: 'security-team', + tier: 'D', + provenance: prov(), + }, + ] + return { contacts, policies: {} } +} diff --git a/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts b/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts index a539f711ed..dcd61f4c9f 100644 --- a/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts +++ b/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts @@ -2,6 +2,7 @@ import { Extractor, ExtractorResult, RawContact, RepoPolicies } from '../../type import { fetchCargo } from './cargo' import { fetchComposer } from './composer' +import { fetchGo } from './go' import { fetchMaven } from './maven' import { fetchNpm } from './npm' import { fetchNuget } from './nuget' @@ -16,7 +17,7 @@ type EcosystemFetcher = ( repoUrl?: string, ) => Promise -// Keyed by the lowercased packages.ecosystem value. go has no package-manifest contacts. +// Keyed by the lowercased packages.ecosystem value. const FETCHERS: Record = { npm: fetchNpm, pypi: fetchPypi, @@ -25,6 +26,7 @@ const FETCHERS: Record = { nuget: fetchNuget, rubygems: fetchRubygems, composer: fetchComposer, + go: fetchGo, } export const extractManifest: Extractor = async (target, deps) => { From 5268362736f6fc807857219041d3cb01fbf301d1 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 21 Jul 2026 14:52:58 +0100 Subject: [PATCH 2/5] feat: add tier D last-resort security contacts (committers + owner) Signed-off-by: Mouad BANI --- .../security-contacts/extractors/repoOwner.ts | 66 +++++++ .../extractors/topCommitters.ts | 184 ++++++++++++++++++ .../src/security-contacts/processBatch.ts | 14 ++ 3 files changed, 264 insertions(+) create mode 100644 services/apps/packages_worker/src/security-contacts/extractors/repoOwner.ts create mode 100644 services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts diff --git a/services/apps/packages_worker/src/security-contacts/extractors/repoOwner.ts b/services/apps/packages_worker/src/security-contacts/extractors/repoOwner.ts new file mode 100644 index 0000000000..ea9c106743 --- /dev/null +++ b/services/apps/packages_worker/src/security-contacts/extractors/repoOwner.ts @@ -0,0 +1,66 @@ +import { getServiceChildLogger } from '@crowd/logging' + +import { parseGithubUrl } from '../../enricher/fetchLightRepo' +import { ExtractorDeps, ProvenanceEntry, RawContact, RepoTarget } from '../types' + +import { isEmail } from './http' + +const log = getServiceChildLogger('security-contacts:repo-owner') + +const SOURCE = 'github-repo-owner' + +export function buildOwnerHandleContact(owner: string, fetchedAt: string, url: string): RawContact { + return { + channel: 'github-handle', + value: owner, + handle: owner, + role: 'org-owner', + tier: 'D', + provenance: [{ source: SOURCE, sourceTier: 'D', path: url, fetchedAt }], + } +} + +export async function fetchRepoOwner( + target: RepoTarget, + deps: Pick, +): Promise { + let owner: string + try { + ;({ owner } = parseGithubUrl(target.url)) + } catch { + return [] + } + + const fetchedAt = new Date().toISOString() + const contacts: RawContact[] = [buildOwnerHandleContact(owner, fetchedAt, target.url)] + + try { + const { text } = await deps.githubGet(`/users/${owner}`) + const email = (text ? (JSON.parse(text) as { email?: unknown }) : null)?.email + if (typeof email === 'string' && isEmail(email)) { + const provenance: ProvenanceEntry[] = [ + { + source: SOURCE, + sourceTier: 'D', + path: `https://api.github.com/users/${owner}`, + fetchedAt, + }, + ] + contacts.push({ + channel: 'email', + value: email, + handle: owner, + role: 'org-owner', + tier: 'D', + provenance, + }) + } + } catch (err) { + log.warn( + { repoId: target.repoId, owner, errMsg: (err as Error).message }, + 'Owner email lookup failed', + ) + } + + return contacts +} diff --git a/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts b/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts new file mode 100644 index 0000000000..b63ba47937 --- /dev/null +++ b/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts @@ -0,0 +1,184 @@ +import { getServiceChildLogger } from '@crowd/logging' + +import { parseGithubUrl } from '../../enricher/fetchLightRepo' +import { ExtractorDeps, ProvenanceEntry, RawContact, RepoTarget } from '../types' + +import { isEmail } from './http' + +const log = getServiceChildLogger('security-contacts:top-committers') + +const SOURCE = 'github-commits' +const SINCE_DAYS = 90 +const TOP_N = 3 +const PER_PAGE = 100 +const MAX_PAGES = 3 + +interface CommitAuthor { + email?: unknown + name?: unknown + date?: unknown +} + +interface CommitEntry { + commit?: { author?: CommitAuthor } + author?: { login?: unknown; type?: unknown } | null +} + +interface AggregatedAuthor { + email: string + name?: string + login?: string + count: number + latestDate: string +} + +const BOT_TOKENS = ['dependabot', 'renovate', 'github-actions'] + +export function isBotCommit(c: CommitEntry): boolean { + if (c.author?.type === 'Bot') return true + const email = typeof c.commit?.author?.email === 'string' ? c.commit.author.email : '' + const name = typeof c.commit?.author?.name === 'string' ? c.commit.author.name : '' + const login = typeof c.author?.login === 'string' ? c.author.login : '' + if (/\[bot\]@/i.test(email)) return true + const haystack = `${email} ${name} ${login}`.toLowerCase() + return BOT_TOKENS.some((token) => haystack.includes(token)) +} + +export function aggregateTopCommitters( + commits: CommitEntry[], + topN: number = TOP_N, +): AggregatedAuthor[] { + const byEmail = new Map() + + for (const c of commits) { + if (isBotCommit(c)) continue + + const rawEmail = c.commit?.author?.email + if (typeof rawEmail !== 'string' || !isEmail(rawEmail)) continue + const email = rawEmail.toLowerCase() + + const rawName = c.commit?.author?.name + const name = typeof rawName === 'string' ? rawName : undefined + const rawDate = c.commit?.author?.date + const date = typeof rawDate === 'string' ? rawDate : undefined + const rawLogin = c.author?.login + const login = typeof rawLogin === 'string' ? rawLogin : undefined + + const existing = byEmail.get(email) + if (existing) { + existing.count += 1 + if (!existing.login && login) existing.login = login + if (date && date > existing.latestDate) existing.latestDate = date + } else { + byEmail.set(email, { + email, + name, + login, + count: 1, + latestDate: date ?? new Date(0).toISOString(), + }) + } + } + + return [...byEmail.values()] + .sort((a, b) => b.count - a.count || a.email.localeCompare(b.email)) + .slice(0, topN) +} + +export function mapCommittersToContacts( + authors: AggregatedAuthor[], + fetchedAt: string, + apiPath: string, +): RawContact[] { + const contacts: RawContact[] = [] + + for (const author of authors) { + const provenance: ProvenanceEntry[] = [ + { + source: SOURCE, + sourceTier: 'D', + path: apiPath, + fetchedAt, + declaredAt: author.latestDate, + }, + ] + + contacts.push({ + channel: 'email', + value: author.email, + name: author.name, + handle: author.login, + role: 'committer', + tier: 'D', + provenance, + }) + + if (author.login) { + contacts.push({ + channel: 'github-handle', + value: author.login, + handle: author.login, + role: 'committer', + tier: 'D', + provenance, + }) + } + } + + return contacts +} + +async function fetchCommitPages( + owner: string, + name: string, + since: string, + deps: Pick, +): Promise { + const commits: CommitEntry[] = [] + + for (let page = 1; page <= MAX_PAGES; page++) { + const path = `/repos/${owner}/${name}/commits?since=${since}&per_page=${PER_PAGE}&page=${page}` + const { text } = await deps.githubGet(path) + if (!text) break + + const entries = JSON.parse(text) as CommitEntry[] + if (!Array.isArray(entries) || entries.length === 0) break + + commits.push(...entries) + if (entries.length < PER_PAGE) break + } + + return commits +} + +export async function fetchTopCommitters( + target: RepoTarget, + deps: Pick, + now: () => Date = () => new Date(), +): Promise { + let owner: string + let name: string + try { + ;({ owner, name } = parseGithubUrl(target.url)) + } catch { + return [] + } + + const since = new Date(now().getTime() - SINCE_DAYS * 24 * 60 * 60 * 1000).toISOString() + const apiPath = `https://api.github.com/repos/${owner}/${name}/commits?since=${since}` + + let commits: CommitEntry[] + try { + commits = await fetchCommitPages(owner, name, since, deps) + } catch (err) { + log.warn( + { repoId: target.repoId, owner, name, errMsg: (err as Error).message }, + 'Commit lookup failed', + ) + return [] + } + + const authors = aggregateTopCommitters(commits) + const fetchedAt = new Date().toISOString() + return mapCommittersToContacts(authors, fetchedAt, apiPath) +} diff --git a/services/apps/packages_worker/src/security-contacts/processBatch.ts b/services/apps/packages_worker/src/security-contacts/processBatch.ts index bbde5eba71..4b0c3e3568 100644 --- a/services/apps/packages_worker/src/security-contacts/processBatch.ts +++ b/services/apps/packages_worker/src/security-contacts/processBatch.ts @@ -1,5 +1,6 @@ import { cancellationSignal, heartbeat } from '@temporalio/activity' +import { classifyEmailReachability } from '@crowd/common' import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { getServiceChildLogger } from '@crowd/logging' @@ -10,10 +11,12 @@ import { mapWithConcurrency } from '../utils/concurrency' import { fetchRepoTree } from './extractors/gitTree' import { extractPvr } from './extractors/pvr' import { extractManifest } from './extractors/registry' +import { fetchRepoOwner } from './extractors/repoOwner' import { extractSecurityContactsFile } from './extractors/securityContactsFile' import { extractSecurityInsights } from './extractors/securityInsights' import { extractSecurityMd } from './extractors/securityMd' import { extractSecurityTxt } from './extractors/securityTxt' +import { fetchTopCommitters } from './extractors/topCommitters' import { githubApiGet } from './githubToken' import { reconcile } from './reconcile' import { deriveGithubHandlesFromNoreplyEmails, resolveCdpEmails } from './resolveCdpEmails' @@ -170,6 +173,17 @@ export async function processRepo( contacts.push(...(await verifyHandleCandidates(target, deps, handleCandidates))) + const isReachableRaw = (c: RawContact): boolean => + c.channel !== 'email' || classifyEmailReachability(c.value).reachable + const hasUsableHigherTierContact = contacts.some((c) => c.tier !== 'D' && isReachableRaw(c)) + if (!hasUsableHigherTierContact) { + const [committers, ownerContacts] = await Promise.all([ + fetchTopCommitters(target, deps), + fetchRepoOwner(target, deps), + ]) + contacts.push(...committers, ...ownerContacts) + } + contacts.push(...deriveGithubHandlesFromNoreplyEmails(contacts)) const handleContacts = contacts.filter((c) => c.channel === 'github-handle') From 0fa7a8e40b14a559af5064719d4d22416fc041e5 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 21 Jul 2026 15:18:58 +0100 Subject: [PATCH 3/5] fix: code review fixes Signed-off-by: Mouad BANI --- .../extractors/topCommitters.ts | 166 +++++++----------- .../src/security-contacts/processBatch.ts | 13 +- .../src/security-contacts/reconcile.ts | 2 +- 3 files changed, 74 insertions(+), 107 deletions(-) diff --git a/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts b/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts index b63ba47937..f6d8f670c1 100644 --- a/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts +++ b/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts @@ -10,113 +10,94 @@ const log = getServiceChildLogger('security-contacts:top-committers') const SOURCE = 'github-commits' const SINCE_DAYS = 90 const TOP_N = 3 -const PER_PAGE = 100 -const MAX_PAGES = 3 -interface CommitAuthor { - email?: unknown - name?: unknown - date?: unknown +interface WeekStat { + w?: unknown + c?: unknown } -interface CommitEntry { - commit?: { author?: CommitAuthor } +interface ContributorStat { author?: { login?: unknown; type?: unknown } | null + weeks?: WeekStat[] } interface AggregatedAuthor { - email: string - name?: string - login?: string + login: string count: number - latestDate: string } const BOT_TOKENS = ['dependabot', 'renovate', 'github-actions'] -export function isBotCommit(c: CommitEntry): boolean { - if (c.author?.type === 'Bot') return true - const email = typeof c.commit?.author?.email === 'string' ? c.commit.author.email : '' - const name = typeof c.commit?.author?.name === 'string' ? c.commit.author.name : '' - const login = typeof c.author?.login === 'string' ? c.author.login : '' - if (/\[bot\]@/i.test(email)) return true - const haystack = `${email} ${name} ${login}`.toLowerCase() - return BOT_TOKENS.some((token) => haystack.includes(token)) +function isBotLogin(login: string): boolean { + const lower = login.toLowerCase() + if (lower.endsWith('[bot]')) return true + return BOT_TOKENS.some((token) => lower.includes(token)) } export function aggregateTopCommitters( - commits: CommitEntry[], + stats: ContributorStat[], + sinceUnixSeconds: number, topN: number = TOP_N, ): AggregatedAuthor[] { - const byEmail = new Map() - - for (const c of commits) { - if (isBotCommit(c)) continue - - const rawEmail = c.commit?.author?.email - if (typeof rawEmail !== 'string' || !isEmail(rawEmail)) continue - const email = rawEmail.toLowerCase() - - const rawName = c.commit?.author?.name - const name = typeof rawName === 'string' ? rawName : undefined - const rawDate = c.commit?.author?.date - const date = typeof rawDate === 'string' ? rawDate : undefined - const rawLogin = c.author?.login - const login = typeof rawLogin === 'string' ? rawLogin : undefined - - const existing = byEmail.get(email) - if (existing) { - existing.count += 1 - if (!existing.login && login) existing.login = login - if (date && date > existing.latestDate) existing.latestDate = date - } else { - byEmail.set(email, { - email, - name, - login, - count: 1, - latestDate: date ?? new Date(0).toISOString(), - }) - } + const authors: AggregatedAuthor[] = [] + + for (const stat of stats) { + if (stat.author?.type === 'Bot') continue + const login = typeof stat.author?.login === 'string' ? stat.author.login : undefined + if (!login || isBotLogin(login)) continue + + const count = (stat.weeks ?? []).reduce((sum, week) => { + const w = typeof week.w === 'number' ? week.w : 0 + const c = typeof week.c === 'number' ? week.c : 0 + return w >= sinceUnixSeconds ? sum + c : sum + }, 0) + if (count > 0) authors.push({ login, count }) } - return [...byEmail.values()] - .sort((a, b) => b.count - a.count || a.email.localeCompare(b.email)) - .slice(0, topN) + return authors.sort((a, b) => b.count - a.count || a.login.localeCompare(b.login)).slice(0, topN) +} + +async function resolvePublicEmail( + login: string, + githubGet: ExtractorDeps['githubGet'], +): Promise { + try { + const { text } = await githubGet(`/users/${login}`) + const email = (text ? (JSON.parse(text) as { email?: unknown }) : null)?.email + return typeof email === 'string' && isEmail(email) ? email : null + } catch (err) { + log.warn({ login, errMsg: (err as Error).message }, 'Committer email resolution failed') + return null + } } -export function mapCommittersToContacts( +export async function mapCommittersToContacts( authors: AggregatedAuthor[], fetchedAt: string, apiPath: string, -): RawContact[] { + githubGet: ExtractorDeps['githubGet'], +): Promise { const contacts: RawContact[] = [] for (const author of authors) { const provenance: ProvenanceEntry[] = [ - { - source: SOURCE, - sourceTier: 'D', - path: apiPath, - fetchedAt, - declaredAt: author.latestDate, - }, + { source: SOURCE, sourceTier: 'D', path: apiPath, fetchedAt }, ] contacts.push({ - channel: 'email', - value: author.email, - name: author.name, + channel: 'github-handle', + value: author.login, handle: author.login, role: 'committer', tier: 'D', provenance, }) - if (author.login) { + const email = await resolvePublicEmail(author.login, githubGet) + if (email) { contacts.push({ - channel: 'github-handle', - value: author.login, + channel: 'email', + value: email, handle: author.login, role: 'committer', tier: 'D', @@ -128,29 +109,6 @@ export function mapCommittersToContacts( return contacts } -async function fetchCommitPages( - owner: string, - name: string, - since: string, - deps: Pick, -): Promise { - const commits: CommitEntry[] = [] - - for (let page = 1; page <= MAX_PAGES; page++) { - const path = `/repos/${owner}/${name}/commits?since=${since}&per_page=${PER_PAGE}&page=${page}` - const { text } = await deps.githubGet(path) - if (!text) break - - const entries = JSON.parse(text) as CommitEntry[] - if (!Array.isArray(entries) || entries.length === 0) break - - commits.push(...entries) - if (entries.length < PER_PAGE) break - } - - return commits -} - export async function fetchTopCommitters( target: RepoTarget, deps: Pick, @@ -164,21 +122,29 @@ export async function fetchTopCommitters( return [] } - const since = new Date(now().getTime() - SINCE_DAYS * 24 * 60 * 60 * 1000).toISOString() - const apiPath = `https://api.github.com/repos/${owner}/${name}/commits?since=${since}` - - let commits: CommitEntry[] + const path = `/repos/${owner}/${name}/stats/contributors` + let stats: ContributorStat[] try { - commits = await fetchCommitPages(owner, name, since, deps) + const { text } = await deps.githubGet(path) + if (!text) return [] + const parsed = JSON.parse(text) + if (!Array.isArray(parsed)) return [] + stats = parsed as ContributorStat[] } catch (err) { log.warn( { repoId: target.repoId, owner, name, errMsg: (err as Error).message }, - 'Commit lookup failed', + 'Contributor stats lookup failed', ) return [] } - const authors = aggregateTopCommitters(commits) + const sinceUnixSeconds = Math.floor(now().getTime() / 1000) - SINCE_DAYS * 24 * 60 * 60 + const authors = aggregateTopCommitters(stats, sinceUnixSeconds) const fetchedAt = new Date().toISOString() - return mapCommittersToContacts(authors, fetchedAt, apiPath) + return mapCommittersToContacts( + authors, + fetchedAt, + `https://api.github.com${path}`, + deps.githubGet, + ) } diff --git a/services/apps/packages_worker/src/security-contacts/processBatch.ts b/services/apps/packages_worker/src/security-contacts/processBatch.ts index 4b0c3e3568..c89832107c 100644 --- a/services/apps/packages_worker/src/security-contacts/processBatch.ts +++ b/services/apps/packages_worker/src/security-contacts/processBatch.ts @@ -18,7 +18,7 @@ import { extractSecurityMd } from './extractors/securityMd' import { extractSecurityTxt } from './extractors/securityTxt' import { fetchTopCommitters } from './extractors/topCommitters' import { githubApiGet } from './githubToken' -import { reconcile } from './reconcile' +import { isJunkContact, reconcile } from './reconcile' import { deriveGithubHandlesFromNoreplyEmails, resolveCdpEmails } from './resolveCdpEmails' import { Extractor, @@ -172,10 +172,13 @@ export async function processRepo( } contacts.push(...(await verifyHandleCandidates(target, deps, handleCandidates))) + contacts.push(...deriveGithubHandlesFromNoreplyEmails(contacts)) - const isReachableRaw = (c: RawContact): boolean => - c.channel !== 'email' || classifyEmailReachability(c.value).reachable - const hasUsableHigherTierContact = contacts.some((c) => c.tier !== 'D' && isReachableRaw(c)) + const isUsableRaw = (c: RawContact): boolean => { + if (isJunkContact(c)) return false + return c.channel !== 'email' || classifyEmailReachability(c.value).reachable + } + const hasUsableHigherTierContact = contacts.some((c) => c.tier !== 'D' && isUsableRaw(c)) if (!hasUsableHigherTierContact) { const [committers, ownerContacts] = await Promise.all([ fetchTopCommitters(target, deps), @@ -184,8 +187,6 @@ export async function processRepo( contacts.push(...committers, ...ownerContacts) } - contacts.push(...deriveGithubHandlesFromNoreplyEmails(contacts)) - const handleContacts = contacts.filter((c) => c.channel === 'github-handle') if (handleContacts.length > 0) { try { diff --git a/services/apps/packages_worker/src/security-contacts/reconcile.ts b/services/apps/packages_worker/src/security-contacts/reconcile.ts index 43206e59fc..edb3e73d77 100644 --- a/services/apps/packages_worker/src/security-contacts/reconcile.ts +++ b/services/apps/packages_worker/src/security-contacts/reconcile.ts @@ -45,7 +45,7 @@ function urlHost(value: string): string | null { } } -function isJunkContact(c: RawContact): boolean { +export function isJunkContact(c: RawContact): boolean { if (c.channel === 'email') { const domain = c.value.split('@')[1]?.toLowerCase().trim() return domain != null && PLACEHOLDER_EMAIL_DOMAINS.has(domain) From f6a517d49a7ae96a0a7c4d84ccd9442e3011aa74 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 21 Jul 2026 15:41:15 +0100 Subject: [PATCH 4/5] fix: 202 status for contributors endpoint Signed-off-by: Mouad BANI --- .../extractors/topCommitters.ts | 67 +++++++++++++++---- .../src/security-contacts/githubToken.ts | 9 ++- .../src/security-contacts/types.ts | 9 ++- 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts b/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts index f6d8f670c1..248da03832 100644 --- a/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts +++ b/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts @@ -10,6 +10,10 @@ const log = getServiceChildLogger('security-contacts:top-committers') const SOURCE = 'github-commits' const SINCE_DAYS = 90 const TOP_N = 3 +// GitHub computes /stats/contributors asynchronously and returns 202 while it's not ready yet — +// poll a few times with a short backoff before giving up for this pass. +const STATS_POLL_ATTEMPTS = 3 +const STATS_POLL_DELAY_MS = 2000 interface WeekStat { w?: unknown @@ -109,10 +113,57 @@ export async function mapCommittersToContacts( return contacts } +async function fetchContributorStats( + path: string, + target: RepoTarget, + owner: string, + name: string, + githubGet: ExtractorDeps['githubGet'], + sleep: (ms: number) => Promise, +): Promise { + for (let attempt = 1; attempt <= STATS_POLL_ATTEMPTS; attempt++) { + let status: number + let text: string | null + try { + ;({ status, text } = await githubGet(path, { extraOkStatuses: [202] })) + } catch (err) { + log.warn( + { repoId: target.repoId, owner, name, errMsg: (err as Error).message }, + 'Contributor stats lookup failed', + ) + return null + } + + if (status === 202) { + if (attempt < STATS_POLL_ATTEMPTS) await sleep(STATS_POLL_DELAY_MS) + continue + } + + if (!text) return null + try { + const parsed = JSON.parse(text) + return Array.isArray(parsed) ? (parsed as ContributorStat[]) : null + } catch (err) { + log.warn( + { repoId: target.repoId, owner, name, errMsg: (err as Error).message }, + 'Contributor stats lookup failed', + ) + return null + } + } + + log.info( + { repoId: target.repoId, owner, name }, + 'Contributor stats still computing after polling — skipping this pass', + ) + return null +} + export async function fetchTopCommitters( target: RepoTarget, deps: Pick, now: () => Date = () => new Date(), + sleep: (ms: number) => Promise = (ms) => new Promise((r) => setTimeout(r, ms)), ): Promise { let owner: string let name: string @@ -123,20 +174,8 @@ export async function fetchTopCommitters( } const path = `/repos/${owner}/${name}/stats/contributors` - let stats: ContributorStat[] - try { - const { text } = await deps.githubGet(path) - if (!text) return [] - const parsed = JSON.parse(text) - if (!Array.isArray(parsed)) return [] - stats = parsed as ContributorStat[] - } catch (err) { - log.warn( - { repoId: target.repoId, owner, name, errMsg: (err as Error).message }, - 'Contributor stats lookup failed', - ) - return [] - } + const stats = await fetchContributorStats(path, target, owner, name, deps.githubGet, sleep) + if (!stats) return [] const sinceUnixSeconds = Math.floor(now().getTime() / 1000) - SINCE_DAYS * 24 * 60 * 60 const authors = aggregateTopCommitters(stats, sinceUnixSeconds) diff --git a/services/apps/packages_worker/src/security-contacts/githubToken.ts b/services/apps/packages_worker/src/security-contacts/githubToken.ts index c118269e96..393de6b53f 100644 --- a/services/apps/packages_worker/src/security-contacts/githubToken.ts +++ b/services/apps/packages_worker/src/security-contacts/githubToken.ts @@ -129,16 +129,19 @@ async function fetchOnce( export async function githubApiGet( path: string, timeoutMs: number, - opts: { raw?: boolean } = {}, + opts: { raw?: boolean; extraOkStatuses?: number[] } = {}, ): Promise { const accept = opts.raw ? 'application/vnd.github.raw' : 'application/vnd.github+json' + const okStatuses = opts.extraOkStatuses + ? new Set([...ABSENT_STATUSES, ...opts.extraOkStatuses]) + : ABSENT_STATUSES const url = `${GITHUB_API}${path}` const resolved = await ensurePool() if (!resolved) { const res = await fetchOnce(url, timeoutMs, { Accept: accept }) if (res.status === 200) return { status: 200, text: await res.text() } - if (ABSENT_STATUSES.has(res.status)) return { status: res.status, text: null } + if (okStatuses.has(res.status)) return { status: res.status, text: null } throw new Error(`githubApiGet ${path} failed: HTTP ${res.status}`) } @@ -166,7 +169,7 @@ export async function githubApiGet( Accept: accept, }) - if (res.status === 200 || ABSENT_STATUSES.has(res.status)) { + if (res.status === 200 || okStatuses.has(res.status)) { pool.parkIfBudgetLow( installationId, numOrNull(res.headers.get('x-ratelimit-remaining')), diff --git a/services/apps/packages_worker/src/security-contacts/types.ts b/services/apps/packages_worker/src/security-contacts/types.ts index af84a0ea6d..26f2fbe79b 100644 --- a/services/apps/packages_worker/src/security-contacts/types.ts +++ b/services/apps/packages_worker/src/security-contacts/types.ts @@ -67,7 +67,7 @@ export interface RepoTarget { export interface GithubGetResult { status: number - /** Null for absent resources (404/410/422/451). */ + /** Null for absent resources (404/410/422/451) or a status listed in extraOkStatuses. */ text: string | null } @@ -75,7 +75,12 @@ export interface ExtractorDeps { fetchTimeoutMs: number /** Required — crates.io rejects requests without an identifying UA. */ userAgent: string - githubGet: (path: string, opts?: { raw?: boolean }) => Promise + /** extraOkStatuses: additional statuses to return as {status, text: null} instead of throwing — + * e.g. 202 for endpoints (like /stats/contributors) that mean "still computing", not absent. */ + githubGet: ( + path: string, + opts?: { raw?: boolean; extraOkStatuses?: number[] }, + ) => Promise /** Default-branch file paths from one git-tree call; null means unresolved — probe as before. */ repoTree: { paths: Set | null } } From 729f6a59873657fd1821a892cb95cef8405de80f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 21 Jul 2026 16:35:23 +0100 Subject: [PATCH 5/5] fix: exclude AI-agent committer accounts from tier D security contacts Signed-off-by: Mouad BANI --- .../extractors/topCommitters.ts | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts b/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts index 248da03832..89ee0ff053 100644 --- a/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts +++ b/services/apps/packages_worker/src/security-contacts/extractors/topCommitters.ts @@ -30,12 +30,26 @@ interface AggregatedAuthor { count: number } -const BOT_TOKENS = ['dependabot', 'renovate', 'github-actions'] +const BOT_TOKENS = [ + 'dependabot', + 'renovate', + 'github-actions', + 'claude', + 'anthropic', + 'copilot', + 'chatgpt', + 'openai', +] + +function matchesBotToken(value: string): boolean { + const lower = value.toLowerCase() + return BOT_TOKENS.some((token) => lower.includes(token)) +} function isBotLogin(login: string): boolean { const lower = login.toLowerCase() if (lower.endsWith('[bot]')) return true - return BOT_TOKENS.some((token) => lower.includes(token)) + return matchesBotToken(lower) } export function aggregateTopCommitters( @@ -68,7 +82,8 @@ async function resolvePublicEmail( try { const { text } = await githubGet(`/users/${login}`) const email = (text ? (JSON.parse(text) as { email?: unknown }) : null)?.email - return typeof email === 'string' && isEmail(email) ? email : null + if (typeof email !== 'string' || !isEmail(email) || matchesBotToken(email)) return null + return email } catch (err) { log.warn({ login, errMsg: (err as Error).message }, 'Committer email resolution failed') return null