From 4a155c14c607df984a628b9ea4a6f869bffc21b9 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 13:52:43 +0200 Subject: [PATCH 01/56] feat: add packagist worker Signed-off-by: anilb --- backend/.env.dist.composed | 3 + backend/.env.dist.local | 3 + .../V1783382400__packagist_worker.sql | 17 + .../0006-packagist-worker-design-decisions.md | 200 ++++++++ docs/adr/README.md | 1 + scripts/builders/packages.env | 2 +- scripts/services/packagist-worker.yaml | 67 +++ services/apps/packages_worker/package.json | 5 + .../apps/packages_worker/src/activities.ts | 15 + .../src/bin/packagist-worker.ts | 8 + .../packages_worker/src/packagist/README.md | 193 ++++++++ .../src/packagist/__tests__/downloads.test.ts | 74 +++ .../packagist/__tests__/dueSelection.test.ts | 251 ++++++++++ .../__tests__/expandMetadata.test.ts | 73 +++ .../packagist/__tests__/fetchPackage.test.ts | 188 ++++++++ .../src/packagist/__tests__/ingest.test.ts | 432 ++++++++++++++++++ .../packagist/__tests__/listPackages.test.ts | 88 ++++ .../src/packagist/__tests__/normalize.test.ts | 260 +++++++++++ .../__tests__/persistMetadata.test.ts | 165 +++++++ .../__tests__/persistPackageInfo.test.ts | 135 ++++++ .../src/packagist/__tests__/wiring.test.ts | 49 ++ .../src/packagist/activities.ts | 392 ++++++++++++++++ .../src/packagist/downloads.ts | 38 ++ .../src/packagist/expandMetadata.ts | 29 ++ .../src/packagist/fetchPackage.ts | 141 ++++++ .../src/packagist/listPackages.ts | 103 +++++ .../src/packagist/normalize.ts | 216 +++++++++ .../src/packagist/retryPolicy.ts | 4 + .../packages_worker/src/packagist/schedule.ts | 72 +++ .../packages_worker/src/packagist/types.ts | 115 +++++ .../src/packagist/upsertMetadata.ts | 90 ++++ .../src/packagist/upsertPackageInfo.ts | 55 +++ .../src/packagist/workflows.ts | 89 ++++ .../src/scripts/triggerPackagistSeed.ts | 82 ++++ .../extractors/registry/index.ts | 4 +- .../packages_worker/src/workflows/index.ts | 6 + .../apps/packages_worker/vitest.config.ts | 42 +- .../src/packages/dependencies.ts | 35 ++ .../data-access-layer/src/packages/index.ts | 2 + .../src/packages/packages.ts | 169 +++++++ .../src/packages/packagistPackageState.ts | 140 ++++++ .../src/packages/versions.ts | 90 ++++ 42 files changed, 4134 insertions(+), 9 deletions(-) create mode 100644 backend/src/osspckgs/migrations/V1783382400__packagist_worker.sql create mode 100644 docs/adr/0006-packagist-worker-design-decisions.md create mode 100644 scripts/services/packagist-worker.yaml create mode 100644 services/apps/packages_worker/src/bin/packagist-worker.ts create mode 100644 services/apps/packages_worker/src/packagist/README.md create mode 100644 services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts create mode 100644 services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts create mode 100644 services/apps/packages_worker/src/packagist/__tests__/expandMetadata.test.ts create mode 100644 services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts create mode 100644 services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts create mode 100644 services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts create mode 100644 services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts create mode 100644 services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts create mode 100644 services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts create mode 100644 services/apps/packages_worker/src/packagist/__tests__/wiring.test.ts create mode 100644 services/apps/packages_worker/src/packagist/activities.ts create mode 100644 services/apps/packages_worker/src/packagist/downloads.ts create mode 100644 services/apps/packages_worker/src/packagist/expandMetadata.ts create mode 100644 services/apps/packages_worker/src/packagist/fetchPackage.ts create mode 100644 services/apps/packages_worker/src/packagist/listPackages.ts create mode 100644 services/apps/packages_worker/src/packagist/normalize.ts create mode 100644 services/apps/packages_worker/src/packagist/retryPolicy.ts create mode 100644 services/apps/packages_worker/src/packagist/schedule.ts create mode 100644 services/apps/packages_worker/src/packagist/types.ts create mode 100644 services/apps/packages_worker/src/packagist/upsertMetadata.ts create mode 100644 services/apps/packages_worker/src/packagist/upsertPackageInfo.ts create mode 100644 services/apps/packages_worker/src/packagist/workflows.ts create mode 100644 services/apps/packages_worker/src/scripts/triggerPackagistSeed.ts create mode 100644 services/libs/data-access-layer/src/packages/dependencies.ts create mode 100644 services/libs/data-access-layer/src/packages/packagistPackageState.ts diff --git a/backend/.env.dist.composed b/backend/.env.dist.composed index b0af8d7a05..efb78e0a52 100644 --- a/backend/.env.dist.composed +++ b/backend/.env.dist.composed @@ -36,5 +36,8 @@ CROWD_PACKAGES_DB_USERNAME=postgres CROWD_PACKAGES_DB_PASSWORD=example CROWD_PACKAGES_DB_DATABASE=packages-db +# Packagist registry crawler contact email +CROWD_PACKAGES_PACKAGIST_MAILTO= + # security-contacts-worker SECURITY_CONTACTS_USER_AGENT="lfx-security-contacts-worker" diff --git a/backend/.env.dist.local b/backend/.env.dist.local index adca4976c3..67a43e009f 100755 --- a/backend/.env.dist.local +++ b/backend/.env.dist.local @@ -179,6 +179,9 @@ CROWD_PACKAGES_DB_USERNAME=postgres CROWD_PACKAGES_DB_PASSWORD=example CROWD_PACKAGES_DB_DATABASE=packages-db +# Packagist registry crawler contact email +CROWD_PACKAGES_PACKAGIST_MAILTO= + # github-repos-enricher ENRICHER_GITHUB_TOKENS= ENRICHER_BATCH_SIZE=100 diff --git a/backend/src/osspckgs/migrations/V1783382400__packagist_worker.sql b/backend/src/osspckgs/migrations/V1783382400__packagist_worker.sql new file mode 100644 index 0000000000..2d8a2e1d5b --- /dev/null +++ b/backend/src/osspckgs/migrations/V1783382400__packagist_worker.sql @@ -0,0 +1,17 @@ + +-- Packagist worker: per-lane ingestion state tracking (metadata, downloads-30d, daily downloads) +CREATE TABLE packagist_package_state ( + purl text PRIMARY KEY, + first_seen_at timestamptz NOT NULL DEFAULT now(), + metadata_last_run_at timestamptz, + metadata_run_result jsonb, -- { status, attempts, httpStatus?, errorKind?, message? } + metadata_last_modified text, -- Last-Modified from p2 endpoint, replayed as If-Modified-Since + downloads_30d_last_run_at timestamptz, + downloads_30d_run_result jsonb, -- { status, attempts, httpStatus?, errorKind?, message? } + daily_downloads_last_run_at timestamptz, + daily_downloads_run_result jsonb -- { status, attempts, httpStatus?, errorKind?, message? } +); + +CREATE INDEX ON packagist_package_state (metadata_last_run_at); +CREATE INDEX ON packagist_package_state (downloads_30d_last_run_at); +CREATE INDEX ON packagist_package_state (daily_downloads_last_run_at); diff --git a/docs/adr/0006-packagist-worker-design-decisions.md b/docs/adr/0006-packagist-worker-design-decisions.md new file mode 100644 index 0000000000..c21a9facb2 --- /dev/null +++ b/docs/adr/0006-packagist-worker-design-decisions.md @@ -0,0 +1,200 @@ +# ADR-0006: Packagist worker — design decisions (living) + +**Date**: 2026-07-13 +**Status**: living +**Deciders**: Anil B + +_Living, consolidated record for the Packagist (PHP/Composer) worker. Record further +Packagist-worker decisions here as new `### subsections` under Decisions rather than opening a +new ADR per decision — mirrors ADR-0001 (oss-packages living ADR) and the ADR-0005 consolidation +note. Each entry is stamped with its own `**Decided**` date; the Changelog at the bottom tracks +when entries are added or revised._ + +## Context + +Packagist is the primary registry for PHP/Composer. Unlike npm/maven/pypi/etc., it is **not +covered by deps.dev** — there is no `PACKAGIST` system in `bigquery-public-data.deps_dev_v1` — so we +cannot seed it from BigQuery the way ADR-0001's universe ingest does for other ecosystems. We crawl +Packagist directly instead (list.json seed → dynamic stats endpoint → static p2 metadata endpoint). +A consequence that shapes several decisions: Packagist hands us **per-package Composer manifests** +(the raw `require` / `require-dev` maps a package declared), **not a pre-resolved dependency graph** +like the one deps.dev exposes in BigQuery. + +## Scope and current status + +| Decision area | Status | +| --------------- | ------- | +| Dependency model | decided | +| Metadata enrichment scope | decided | +| Lane architecture & cadence | decided | + +--- + +## Decisions + +### Dependency model: direct edges + declared constraints, resolved at query time + +We store, in `package_dependencies`, **only the direct dependency edges a version declares** — one +row per `(version_id, depends_on_id, dependency_kind)` — and we **resolve concrete versions and +transitive closure at query time**, never at ingest. + +Per stored edge (for the critical slice only — dependency edges ride the same p2 metadata fetch as +versions, which we only run for critical packages): + +- `require` → `dependency_kind = 'direct'`, `require-dev` → `dependency_kind = 'dev'`. +- `version_constraint` = the **declared** constraint string verbatim (e.g. `^3.0 || ^2.0`). +- `depends_on_id` = the depended-on **package** row (resolved by name; edges whose target isn't a + known packages row are counted and skipped, not errored). +- `depends_on_version_id` = **NULL** — we do not resolve the concrete satisfying version at ingest. +- Platform requirements (`php`, `hhvm`, `ext-*`, `lib-*`, `composer-*` — anything with no `/`) are + excluded; they aren't packages. + +**Provenance.** The query-time transitive model is a **stated requirement**, quoted verbatim from +the Packagist change brief: + +> We store only direct dependencies (the requirements declared in each version's composer.json) and +> walk the tree at query time when transitive resolution is needed. We do not need to compute or +> store the resolved graph at ingest. + +Leaving `depends_on_version_id` NULL is the column-level **implementation** of that last sentence — +the resolved concrete version *is* part of "the resolved graph" — locked at the plan-approval gate +and consistent with the pre-existing schema comment (`depends_on_version_id bigint, -- resolved +version; NULL if unknown`). Storing the declared `version_constraint` but not a pinned version means +the two derived views — *which version a constraint resolves to* and *the full transitive tree* — +are both computed on read. + +**Why not pin `depends_on_version_id` at ingest (as deps.dev does):** a resolved version is *derived +and volatile*. `^3.0` resolves to whatever `psr/log` patch is newest this week; a stored pin goes +stale the moment a patch ships and would need continuous rewriting. deps.dev's own worker documents +exactly this churn — re-resolving `^4.17.0` every snapshot is what produced its ~555M-row edge +exports and forced a snapshot-diff to suppress it. We sidestep it entirely by storing the immutable +declared constraint and resolving on demand. deps.dev pins because it *already* has a fully-resolved +graph in BigQuery for free; we only get manifests, so resolving eagerly would mean building and +continuously re-running a Composer resolver for no durable benefit. + +**Why store `dev` edges when deps.dev doesn't:** deps.dev's resolved graph is runtime-only — every +edge lands as `'direct'`, it has no dev/peer notion. Because we read the raw manifest we can split +`require-dev` into `'dev'`, so Packagist edges are *richer per-edge* (dev deps + declared +constraints) even though they are *shallower* (no resolved target version) and *narrower* (critical +slice only) than deps.dev's. + +**Consequences.** + +_Positive:_ + +- Write path is small and touches only immutable declared facts — no re-resolution churn, no + billion-row resolved-graph materialization for Packagist. +- Query-time resolution and transitive walks always reflect current data (new patches, newly-added + deep dependencies) with no re-ingest. +- Captures dev-dependencies (`kind='dev'`), a dimension deps.dev-seeded ecosystems lack. +- Schema-compatible with the deps.dev-populated `package_dependencies`: same table, same unique key + `(version_id, depends_on_id, dependency_kind)`; the `depends_on_id` HASH-partitioning (ADR-0001) + keeps the upstream "who depends on X?" hot path fast for Packagist edges too. + +_Negative / trade-offs:_ + +- `depends_on_version_id` is NULL for every Packagist edge — a consumer that wants the exact version + a dependency pulls in must resolve the constraint itself; there is no per-edge pinned version the + way there is for npm/maven/pypi/cargo seeded from deps.dev. +- Transitive questions ("everything X depends on") pay a recursive walk over direct edges at read + time rather than a single indexed lookup. +- Dependency edges exist for the **critical slice only** — non-critical Packagist packages have + none, because the p2 metadata crawl that produces them is critical-scoped. + +_Risks:_ + +- If an LFX product later needs per-edge **resolved** versions for Packagist, that is a + **requirements change** — it reopens the "do not store the resolved graph at ingest" decision and + would require standing up a Composer-style resolver (and accepting the re-resolution churn deps.dev + fights). Flagged here so the trade-off is revisited deliberately, not silently. + +**Decided**: 2026-07-13 + +--- + +### Metadata enrichment scope: all packages, not the critical slice + +The metadata lane (p2 manifests → `versions`, `package_dependencies` edges, and the package-level +aggregates `licenses` / `latest_version` / `versions_count` / `first_release_at` / +`latest_release_at` / `homepage`) runs for **all ~500K Packagist packages**, not only the critical +slice: `CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL` defaults to `false` (setting it `true` +narrows back to the critical slice). The same lane additionally links `repos` / +`package_repos` rows for all packages (`'declared'` @ `0.8`, the manifest-declared convention +shared with npm/pypi/maven/cargo), so the shared github-repos-enricher picks Packagist repos up. + +**Provenance.** Requested in the deps.dev-parity review (Joana): deps.dev has zero Packagist data, +so unlike every other ecosystem there is no external source backfilling the non-critical majority — +the tiering left most of the catalog thin for no data-availability reason. The goal is best-effort +parity with what deps.dev populates for its ecosystems. + +**Why this deliberately diverges from pypi:** the pypi sibling documents critical-only as the +"intended steady state" with all-packages as a temporary bootstrap mode. That design assumes +deps.dev provides package-level data for the whole ecosystem, so per-package registry enrichment +only needs to deepen the critical slice. Packagist has no such backstop; the registry crawl *is* +the universe source. The p2 endpoint is a static, CDN-served file designed to be mirrored (it is +how Composer clients resolve), with `If-Modified-Since`/304 replay — so the steady-state weekly +cost after the first full pass is dominated by 304s, not re-parses. + +**What stays critical-only (deliberate, not parity gaps):** `downloads_daily` and +maintainers. deps.dev carries no downloads at all, and npm/pypi both scope daily downloads and +maintainer enrichment to the critical slice; ungating maintainers would also cost ~3–4M per-row +DB round-trips per weekly cycle (`upsertPackageMaintainers` is 2M+2 queries per package) — a +batching rewrite is a prerequisite before that scope can ever widen. + +**Consequences.** + +_Positive:_ versions, direct dependency edges, licenses/latest-version/release-date aggregates, +homepage, and repo links exist for the whole catalog — matching deps.dev's coverage shape; ranking +inputs and downstream consumers see a fully-populated ecosystem rather than a thin one. + +_Negative / trade-offs:_ the weekly metadata drain walks ~500K purls (~10k keyset batches, ~500 +continueAsNew generations) instead of ~56K; `versions` grows by an estimated 5–10M rows and +`package_dependencies` by ~20–40M edges (both well within the tables' 90M+/1.15B+ design points); +the two packagist workers now default to opposite scopes, mitigated by explicit comments at both +flip points. + +_Risks:_ the all-packages due-selection abandons the partial `is_critical` index and rides the +`purl` btree keyset walk — late-cycle batches skip progressively more already-scanned rows; if the +tail slows measurably, revisit with an `EXPLAIN` (flagged in the rollout checklist). + +**Decided**: 2026-07-16 + +--- + +### Lane architecture: seed → chained enrichment, downloads as dedicated lanes + +Four workflows instead of the earlier seed/stats/metadata split (the `ingestPackagistStats` +workflow, whose name and mixed responsibilities were unclear, was removed): + +1. **`seedPackagistPackages`** — weekly cron. Discovery only; on completion **chain-starts the + enrichment drain as a child workflow** (`ParentClosePolicy.ABANDON`) instead of a second cron, + so "after seed" is an event, not a clock offset (same idiom as `bootstrapOsspckgs`). +2. **`ingestPackagistMetadata`** — the merged enrichment lane: per package it fetches **both** + registry endpoints — dynamic (package info, counters, repo link, maintainers) and p2 (versions, + dependencies, aggregates, homepage) — under one `metadata_last_run_at` watermark. +3. **`ingestPackagistDownloads30d`** — monthly cron **on the 1st**: captures the observed rolling + 30d value as the month's `downloads_last_30d` window row (mirrored to the packages column), + npm's breadth pattern (run-scoped cutoff, per-purl watermark). Running on the boundary replaces + the earlier "first scan on/after the 1st" heuristic. No history/backfill lane — Packagist keeps + no historical series, so a missed month is unrecoverable by design. +4. **`ingestPackagistDownloadsDaily`** — daily cron, critical slice only. + +Each lane owns its own watermark + run result in `packagist_package_state` +(`metadata_* / downloads_30d_* / daily_downloads_*`; the ambiguous `p2_last_modified` renamed to +`metadata_last_modified`). Trade-offs: the enrichment lane makes two HTTP calls per package +(bounded by the dynamic endpoint's 10-concurrency), the dynamic endpoint is fetched by three lanes +at different cadences (~500K extra fetches/month for the 30d lane), and a hard seed failure skips +that week's enrichment (recoverable via the manual trigger). + +**Decided**: 2026-07-16 + +--- + +## Changelog + +- **2026-07-13** — ADR created. First entry: _Dependency model: direct edges + declared constraints, + resolved at query time_. +- **2026-07-16** — Added _Metadata enrichment scope: all packages, not the critical slice_ (also + covers repo linking for all packages and the deliberate critical-only carve-outs). +- **2026-07-16** — Added _Lane architecture: seed → chained enrichment, downloads as dedicated + lanes_ (removes the stats lane; renames `p2_last_modified` → `metadata_last_modified`). diff --git a/docs/adr/README.md b/docs/adr/README.md index 250066a4a4..ca32031ed9 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -12,6 +12,7 @@ Use the `/adr` skill in Claude Code to record new ADRs or query past decisions. | [ADR-0003](./0003-deps-bq-table-selection.md) | Use DependencyGraphEdgesLatest for deps ingestion; defer DependenciesLatest until NUGET or GO needed | accepted | 2026-05-29 | | [ADR-0004](./0004-go-nuget-transitive-dependent-counts.md) | Compute GO/NUGET transitive dependent counts via exact reverse closure (over HLL approximation) | accepted | 2026-06-23 | | [ADR-0005](./0005-pypi-downloads-bigquery-merge-scoping.md) | PyPI downloads via BigQuery bulk export, scoped in the Postgres merge | accepted | 2026-07-01 | +| [ADR-0006](./0006-packagist-worker-design-decisions.md) | Packagist worker — design decisions (living) | living | 2026-07-13 | ## Why ADRs? diff --git a/scripts/builders/packages.env b/scripts/builders/packages.env index d178e59ec3..0f11ead775 100644 --- a/scripts/builders/packages.env +++ b/scripts/builders/packages.env @@ -1,4 +1,4 @@ DOCKERFILE="./services/docker/Dockerfile.packages" CONTEXT="../" REPO="sjc.ocir.io/axbydjxa5zuh/packages" -SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker" +SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker packagist-worker" diff --git a/scripts/services/packagist-worker.yaml b/scripts/services/packagist-worker.yaml new file mode 100644 index 0000000000..3956a8c51a --- /dev/null +++ b/scripts/services/packagist-worker.yaml @@ -0,0 +1,67 @@ +version: '3.1' + +x-env-args: &env-args + DOCKER_BUILDKIT: 1 + NODE_ENV: docker + SERVICE: packagist-worker + CROWD_TEMPORAL_TASKQUEUE: packagist-worker + CROWD_TEMPORAL_NAMESPACE: ${CROWD_PACKAGES_TEMPORAL_NAMESPACE} + SHELL: /bin/sh + SUPPRESS_NO_CONFIG_WARNING: 'true' + +services: + packagist-worker: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.packages + command: 'pnpm run start:packagist-worker' + working_dir: /usr/crowd/app/services/apps/packages_worker + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + restart: always + networks: + - crowd-bridge + + packagist-worker-dev: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.packages + command: 'pnpm run dev:packagist-worker' + working_dir: /usr/crowd/app/services/apps/packages_worker + # user: '${USER_ID}:${GROUP_ID}' + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + hostname: packagist-worker + networks: + - crowd-bridge + volumes: + - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src + - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src + - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src + - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src + - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src + - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src + - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src + - ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src + - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src + - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src + - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src + - ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src + - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src + - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src + - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src + - ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src + +networks: + crowd-bridge: + external: true diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json index d500ab3301..3e6ddc6e3a 100644 --- a/services/apps/packages_worker/package.json +++ b/services/apps/packages_worker/package.json @@ -25,6 +25,8 @@ "trigger-go:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=go-worker tsx src/scripts/triggerGoEnrich.ts", "trigger-nuget": "SERVICE=nuget-worker tsx src/scripts/triggerNuGetSync.ts", "trigger-nuget:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=nuget-worker tsx src/scripts/triggerNuGetSync.ts", + "trigger-packagist": "SERVICE=packagist-worker tsx src/scripts/triggerPackagistSeed.ts", + "trigger-packagist:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=packagist-worker tsx src/scripts/triggerPackagistSeed.ts", "start:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker tsx src/bin/npm-worker.ts", "dev:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts", "dev:npm-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts", @@ -49,6 +51,9 @@ "start:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker tsx src/bin/nuget-worker.ts", "dev:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts", "dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts", + "start:packagist-worker": "CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker tsx src/bin/packagist-worker.ts", + "dev:packagist-worker": "CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/packagist-worker.ts", + "dev:packagist-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/packagist-worker.ts", "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts", "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts", "import:maven-maintainers": "SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts", diff --git a/services/apps/packages_worker/src/activities.ts b/services/apps/packages_worker/src/activities.ts index b747a3bd3d..a6929db29a 100644 --- a/services/apps/packages_worker/src/activities.ts +++ b/services/apps/packages_worker/src/activities.ts @@ -33,3 +33,18 @@ export { export { getCriticalPypiCount } from './pypi/downloads/getCriticalPypiCount' export { processNuGetBatch } from './nuget/activities' export { processSecurityContactsBatch } from './security-contacts/activities' +export { + ingestOnePackagistMetadata, + ingestOnePackagist30dWindow, + ingestOnePackagistDailyDownload, + runPackagistPackageSeed, + getPackagistMetadataBatch, + ingestPackagistMetadataBatch, + getPackagist30dBatch, + ingestPackagist30dBatch, + getPackagistDailyBatch, + ingestPackagistDailyBatch, + getCriticalPackagistCount, + packagistCurrentTimestamp, + packagistStopAfterFirstPage, +} from './packagist/activities' diff --git a/services/apps/packages_worker/src/bin/packagist-worker.ts b/services/apps/packages_worker/src/bin/packagist-worker.ts new file mode 100644 index 0000000000..0e7e23670a --- /dev/null +++ b/services/apps/packages_worker/src/bin/packagist-worker.ts @@ -0,0 +1,8 @@ +import { schedulePackagistIngest } from '../packagist/schedule' +import { svc } from '../service' + +setImmediate(async () => { + await svc.init() + await schedulePackagistIngest() + await svc.start() +}) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md new file mode 100644 index 0000000000..fc17d11d26 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/README.md @@ -0,0 +1,193 @@ +# Packagist Worker — Workflows + +The Packagist worker keeps the PHP/Composer slice of the packages database fresh +by crawling **packagist.org directly** — deps.dev has no Packagist coverage, so +unlike npm/maven/pypi there is no BigQuery universe to import from; the registry +crawl *is* the universe source. It runs **four Temporal workflows** on the +`packagist-worker` task queue: three cron schedules registered at worker boot +(`src/bin/packagist-worker.ts`), plus the metadata drain, which the seed chains +as a child workflow on completion. + +Identity: ecosystem `packagist`, purls `pkg:composer/{vendor}/{name}` +(namespace = vendor). Audit tag: `packagist` in `audit_field_changes`. + +Shared design notes: + +- **Two registry endpoints.** The *dynamic* endpoint + (`packagist.org/packages/{vendor}/{name}.json`, server-cached ~12h) carries + package info and live counters: downloads, dependents, description, repo URL, + abandoned flag, maintainers. The *static* p2 endpoint + (`repo.packagist.org/p2/{vendor}/{name}.json`, CDN-served, the same file + Composer clients resolve) carries the version manifests: every tagged + release with its `require`/`require-dev`, licenses, homepage. +- **Politeness.** Bounded concurrency per Packagist's guidelines — max 10 + in-flight dynamic requests (their published limit; running hotter gets + connections reset). Since every lane's ingest starts with a dynamic fetch, + all lanes run at that cap (env-tunable downward, hard-capped at 10); the + p2 fetches inside the metadata lane inherit it, well under p2's own 20. + User-Agent carries a `mailto=` contact + (`CROWD_PACKAGES_PACKAGIST_MAILTO`). Cron minutes are deliberately off `:00`. + p2 fetches replay the stored `Last-Modified` as `If-Modified-Since`; a `304` + records the run without re-downloading or re-parsing. +- **Watermark-driven drains.** Each lane tracks its own per-purl watermark + + run result in `packagist_package_state` (`metadata_last_run_at` / + `downloads_30d_last_run_at` / `daily_downloads_last_run_at`, each with a + `*_run_result` JSONB). Workflows drain in `continueAsNew` rounds (20 × 50), + so an interrupted run resumes where it left off on the next launch. +- **Error handling.** 404 → mark the lane's watermark with an error result, + skip, don't retry. Malformed body/shape → 3 fast in-lane retries (1s linear + backoff), then mark errored and move on. 429/5xx/network/timeout → throw and + ride Temporal's exponential activity retry (5 attempts, lockstep with the + per-item give-up so one bad package can never stall a drain). +- **Tier scoping.** Discovery, package info, repo links, versions, and + dependencies cover **all** packages (deliberate divergence from pypi — see + ADR-0006). Only `downloads_daily` and maintainers are + **critical-slice-only**, matching npm/pypi. + +--- + +## 1. `seedPackagistPackages` — universe discovery + +**Schedule:** `packagist-seed`, weekly Sunday `02:17` UTC. +**Targets:** the full catalog (~500K names) — criticality not consulted. + +**What it does** (`runPackagistPackageSeed`): one GET of +`packagist.org/packages/list.json`, validates/lowercases/dedupes the names, +inserts identity rows in 5,000-row chunks, then **chain-starts the metadata +drain** (child workflow, `ParentClosePolicy.ABANDON`). + +**Populates:** `packages` only — `purl`, `ecosystem='packagist'`, +`namespace` (vendor), `name`, `registry_url`, `status='active'`, +`ingestion_source='packagist-registry'`. `ON CONFLICT (purl) DO NOTHING`: +re-runs only add newly published packages and never touch enriched rows. + +--- + +## 2. `ingestPackagistMetadata` — the enrichment drain (both endpoints) + +**Schedule:** none of its own — **chained off the seed** (effectively weekly, +Sunday right after `02:17`), so newly published packages exist as rows before +dependency targets are resolved. Runs independently once started; a failed +seed skips that week's drain (recover with +`pnpm trigger-packagist:local metadata`). +**Targets:** **all** packagist packages due for a refresh — +`CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL` defaults to `false` +(`true` narrows to the critical slice). Due = never scanned or +`metadata_last_run_at` older than 7 days (`…_METADATA_REFRESH_DAYS`). + +**What it does:** per package, **two fetches** — the dynamic endpoint (≤10 +concurrent) and the p2 endpoint (with `If-Modified-Since`; steady state is +mostly 304s). A package the dynamic endpoint 404s is marked errored without +touching p2. + +**Populates, per scanned package:** + +- From the **dynamic endpoint**: + - `packages` — `description`, `declared_repository_url`, `repository_url` + (canonicalized), `status` (`abandoned` → `deprecated`, else `active`), + `downloads_last_30d` (rolling monthly), `total_downloads`, + `dependent_count` (registry-reported direct dependents — a ranking input), + `last_synced_at`. Counters COALESCE: a missing value never wipes a known one. + - `repos` + `package_repos` — **all packages**: canonicalized repo URL linked + with `source='declared'`, `confidence=0.8` (the manifest-declared convention + shared with npm/pypi/maven/cargo); feeds the shared github-repos-enricher. + Skipped when no URL or it can't be canonicalized. + - `maintainers` + `package_maintainers` — **critical only**: usernames from + the dynamic endpoint (`role='maintainer'`). +- From the **p2 endpoint**: + - `versions` — one row per tagged release (`number`, `published_at`, + `is_prerelease`, `is_latest`, `licenses`); dev branches skipped. + - `package_dependencies` — direct edges only (`require` → `'direct'`, + `require-dev` → `'dev'`), declared constraints verbatim, platform packages + excluded, `depends_on_version_id` NULL (resolved at query time — ADR-0006). + - `packages` aggregates — `versions_count`, `latest_version`, + `first_release_at`, `latest_release_at`, `licenses`, `homepage` — all + written non-destructively (COALESCE / keep-on-zero). +- `packagist_package_state` — `metadata_last_run_at`, `metadata_run_result`, + and `metadata_last_modified` (the p2 `Last-Modified`, replayed as + `If-Modified-Since` next run). +- `audit_field_changes` — every changed field above, worker tag `packagist`. + +--- + +## 3. `ingestPackagistDownloads30d` — monthly rolling-window capture + +**Schedule:** `packagist-downloads-30d`, monthly on the **1st** at `03:53` UTC — +anchored on the month boundary so the observed rolling-30d value approximates +the calendar month. +**Targets:** **all** packagist packages, npm-style breadth: the run fixes a +`cutoff` timestamp once, and every package whose `downloads_30d_last_run_at` +is older (or null) is due exactly once per run. + +**What it does:** fetches the dynamic endpoint per package and records +`downloads.monthly` as the month's window. + +**Populates:** one `downloads_last_30d` row per purl per month +(`end_date` = the 1st, `start_date` = end − 30d), **mirrored to +`packages.downloads_last_30d`**. A window already recorded for the month is +never overwritten — the first observation (closest to the boundary) wins. +Packagist keeps no history, so unlike npm there is no backfill lane; a missed +month stays missed. State: `downloads_30d_last_run_at` + `downloads_30d_run_result`. + +--- + +## 4. `ingestPackagistDownloadsDaily` — daily capture (critical slice) + +**Schedule:** `packagist-downloads-daily`, daily `06:23` UTC. +**Targets:** `is_critical = TRUE` only; no-ops (guard) while zero packagist +rows are critical — i.e. until the first `rank_packages()` pass. Same cutoff +watermark pattern as the 30d lane. + +**What it does:** fetches the dynamic endpoint per critical package and records +the registry's rolling daily figure. + +**Populates:** one `downloads_daily` row per package per day +(`package_id`, run date, count). State: `daily_downloads_last_run_at` + +`daily_downloads_run_result`. + +--- + +## What this worker deliberately does NOT write + +- `transitive_dependent_count`, `dependent_repos_count` — not computable from + the registry; needs a reverse-closure over our stored direct edges + (future work, see ADR-0006 risks). +- Advisories — the OSV worker owns security data platform-wide. +- `is_critical` / `criticality_score` / ranking columns — the shared + criticality worker; this worker only *reads* `is_critical` for scoping. +- Repo stars/forks/scorecard — the shared github-repos-enricher, fed by the + `package_repos` links this worker creates. +- `keywords`, `versions.is_yanked`, `depends_on_version_id` — see above. + +## Configuration + +All optional; defaults in `activities.ts` / `fetchPackage.ts`: + +| Env var (`CROWD_PACKAGES_PACKAGIST_…`) | Default | Meaning | +|---|---|---| +| `MAILTO` | `oss-packages@linuxfoundation.org` | Contact in the crawl User-Agent | +| `STATS_CONCURRENCY` | 10 (hard cap 10) | Parallel package ingests across all lanes — every lane starts with a dynamic-endpoint fetch, so the stricter 10-limit bounds everything | +| `METADATA_REFRESH_DAYS` | 7 | Metadata refresh window | +| `RUN_ONLY_FOR_CRITICAL` | `false` | `true` narrows metadata to the critical slice | +| `STOP_AFTER_FIRST_PAGE` | `false` | Debug: one drain page, no continueAsNew | + +## Running it + +```bash +# local worker (hot reload; needs packages-db + temporal from scripts/scaffold.yaml) +DEV=1 ./scripts/cli service packagist-worker up + +# trigger on demand instead of waiting for the crons +cd services/apps/packages_worker +pnpm trigger-packagist:local seed # discovery (chain-starts metadata!) +pnpm trigger-packagist:local metadata # enrichment: info + versions + deps +pnpm trigger-packagist:local downloads-30d # monthly rolling-window capture +pnpm trigger-packagist:local downloads-daily # daily capture, critical slice +``` + +Note: triggering `seed` also chain-starts the full `metadata` drain (set +`CROWD_PACKAGES_PACKAGIST_STOP_AFTER_FIRST_PAGE=true` locally to bound it). +Local smoke order: seed → metadata → (rank) → downloads lanes for +critical-scoped writes. State lives in `packagist_package_state` +(migration `V1783382400__packagist_worker.sql`); design decisions in +`docs/adr/0006-packagist-worker-design-decisions.md`. diff --git a/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts b/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts new file mode 100644 index 0000000000..134031d129 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts @@ -0,0 +1,74 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + getExistingLast30dEndDates, + upsertLast30dDownload, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { monthlyWindowFor, persistPackagist30dWindow } from '../downloads' + +vi.mock('@crowd/data-access-layer/src/packages', () => ({ + getExistingLast30dEndDates: vi.fn().mockResolvedValue([]), + upsertLast30dDownload: vi.fn().mockResolvedValue([]), +})) + +const mockExistingWindows = vi.mocked(getExistingLast30dEndDates) +const mockWindow = vi.mocked(upsertLast30dDownload) + +const qx = {} as QueryExecutor +const PURL = 'pkg:composer/monolog/monolog' + +beforeEach(() => { + vi.clearAllMocks() + mockExistingWindows.mockResolvedValue([]) +}) + +// Observed rolling window anchored on the 1st of the run month, +// start_date = end_date − 30 days. +describe('monthlyWindowFor', () => { + it('anchors the window on the 1st of the run month', () => { + expect(monthlyWindowFor('2026-07-15')).toEqual({ + startDate: '2026-06-01', + endDate: '2026-07-01', + }) + expect(monthlyWindowFor('2026-07-01')).toEqual({ + startDate: '2026-06-01', + endDate: '2026-07-01', + }) + }) + + it('computes start_date by calendar arithmetic, not "previous month 1st"', () => { + expect(monthlyWindowFor('2026-03-05')).toEqual({ + startDate: '2026-01-30', + endDate: '2026-03-01', + }) + }) +}) + +// The monthly downloads-30d lane: one window row per purl per month, mirrored to +// packages.downloads_last_30d (npm parity). A window already recorded for the month +// is never overwritten — the value is the observation closest to the boundary. +describe('persistPackagist30dWindow', () => { + it('writes the window with the mirror when the month has no row yet', async () => { + await persistPackagist30dWindow(qx, PURL, 300, '2026-07-01') + + expect(mockExistingWindows).toHaveBeenCalledWith(qx, PURL, '2026-07-01', '2026-07-01') + expect(mockWindow).toHaveBeenCalledWith(qx, PURL, '2026-06-01', '2026-07-01', 300, true) + }) + + it('does not overwrite an existing window for the month', async () => { + mockExistingWindows.mockResolvedValue(['2026-07-01']) + + await persistPackagist30dWindow(qx, PURL, 300, '2026-07-15') + + expect(mockWindow).not.toHaveBeenCalled() + }) + + it('writes nothing when the registry reported no monthly count', async () => { + await persistPackagist30dWindow(qx, PURL, null, '2026-07-01') + + expect(mockExistingWindows).not.toHaveBeenCalled() + expect(mockWindow).not.toHaveBeenCalled() + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts new file mode 100644 index 0000000000..a9119a078f --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts @@ -0,0 +1,251 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + getCriticalPackagistPackageCount, + insertPackagistPackages, + updatePackagistPackageStats, +} from '@crowd/data-access-layer/src/packages/packages' +import { + getPackagist30dDuePurls, + getPackagistDailyDownloadsDue, + getPackagistMetadataDuePurls, + markPackagist30dProcessed, + markPackagistDailyProcessed, + markPackagistMetadataScanned, +} from '@crowd/data-access-layer/src/packages/packagistPackageState' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +// Executes the real DAL functions against a capturing stand-in executor — no database. +function makeQx() { + return { + select: vi.fn().mockResolvedValue([]), + selectOne: vi.fn().mockResolvedValue({ count: '0' }), + selectOneOrNone: vi.fn().mockResolvedValue(null), + result: vi.fn().mockResolvedValue(0), + tx: vi.fn(), + } +} +type FakeQx = ReturnType +const asQx = (qx: FakeQx) => qx as unknown as QueryExecutor + +let qx: FakeQx + +beforeEach(() => { + qx = makeQx() +}) + +// Metadata (merged enrichment) due-selection: keyset pagination, refresh window, and +// the stored Last-Modified for If-Modified-Since replay on the p2 fetch. +describe('getPackagistMetadataDuePurls', () => { + it('returns purls with their stored Last-Modified, scoped by refresh window', async () => { + qx.select.mockResolvedValue([ + { purl: 'pkg:composer/a/x', metadata_last_modified: 'Tue, 30 Jun 2026 00:00:00 GMT' }, + { purl: 'pkg:composer/b/y', metadata_last_modified: null }, + ]) + + const candidates = await getPackagistMetadataDuePurls(asQx(qx), '', 50, 7, true) + + expect(candidates).toEqual([ + { purl: 'pkg:composer/a/x', metadataLastModified: 'Tue, 30 Jun 2026 00:00:00 GMT' }, + { purl: 'pkg:composer/b/y', metadataLastModified: null }, + ]) + const [sql, params] = qx.select.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + expect(sql).toMatch(/is_critical/) + expect(sql).toMatch(/metadata_last_run_at/) + expect(sql).toMatch(/ORDER BY\s+p\.purl/i) + expect(params).toMatchObject({ batchSize: 50, refreshDays: 7, onlyCritical: true }) + }) + + it('selects across all packagist packages when onlyCritical is false (the all-packages default)', async () => { + await getPackagistMetadataDuePurls(asQx(qx), '', 50, 7, false) + + const [sql, params] = qx.select.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + expect(params).toMatchObject({ onlyCritical: false }) + }) +}) + +describe('markPackagistMetadataScanned', () => { + it('upserts the metadata run result and stores a fresh Last-Modified when given', async () => { + await markPackagistMetadataScanned( + asQx(qx), + 'pkg:composer/a/x', + { status: 'success', attempts: 1 }, + 'Wed, 01 Jul 2026 00:00:00 GMT', + ) + + const [sql, params] = qx.result.mock.calls[0] + expect(sql).toMatch(/packagist_package_state/) + expect(sql).toMatch(/metadata_run_result/) + expect(sql).toMatch(/metadata_last_modified/) + expect(sql).toMatch(/ON CONFLICT \(purl\)/) + expect(params).toMatchObject({ + purl: 'pkg:composer/a/x', + result: JSON.stringify({ status: 'success', attempts: 1 }), + metadataLastModified: 'Wed, 01 Jul 2026 00:00:00 GMT', + }) + }) + + it('keeps the stored Last-Modified when none is given', async () => { + await markPackagistMetadataScanned(asQx(qx), 'pkg:composer/a/x', { + status: 'success', + attempts: 1, + }) + + const [sql, params] = qx.result.mock.calls[0] + expect(params).toMatchObject({ metadataLastModified: null }) + expect(sql).toMatch(/COALESCE/i) + }) +}) + +// Monthly downloads-30d lane: npm-style breadth — every package is due once per run +// (watermark older than the run's cutoff, or never run). +describe('getPackagist30dDuePurls / markPackagist30dProcessed', () => { + it('selects packagist purls whose 30d watermark is older than the cutoff', async () => { + qx.select.mockResolvedValue([{ purl: 'pkg:composer/a/x' }]) + + const purls = await getPackagist30dDuePurls(asQx(qx), '2026-07-01T03:53:00Z', 50) + + expect(purls).toEqual(['pkg:composer/a/x']) + const [sql, params] = qx.select.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + expect(sql).toMatch(/downloads_30d_last_run_at/) + expect(sql).toMatch(/LIMIT/i) + expect(params).toMatchObject({ cutoff: '2026-07-01T03:53:00Z', batchSize: 50 }) + }) + + it('bumps the 30d watermark with the run result', async () => { + await markPackagist30dProcessed(asQx(qx), 'pkg:composer/a/x', { + status: 'success', + attempts: 1, + }) + + const [sql, params] = qx.result.mock.calls[0] + expect(sql).toMatch(/packagist_package_state/) + expect(sql).toMatch(/downloads_30d_last_run_at/) + expect(sql).toMatch(/downloads_30d_run_result/) + expect(sql).toMatch(/ON CONFLICT \(purl\)/) + expect(params).toMatchObject({ + purl: 'pkg:composer/a/x', + result: JSON.stringify({ status: 'success', attempts: 1 }), + }) + }) +}) + +// Daily downloads lane: critical slice only, cutoff watermark, returns the package id +// the downloads_daily insert needs. +describe('getPackagistDailyDownloadsDue / markPackagistDailyProcessed', () => { + it('selects critical packagist packages due for a daily capture, with their ids', async () => { + qx.select.mockResolvedValue([{ purl: 'pkg:composer/a/x', package_id: '7' }]) + + const due = await getPackagistDailyDownloadsDue(asQx(qx), '2026-07-15T06:23:00Z', 50) + + expect(due).toEqual([{ purl: 'pkg:composer/a/x', packageId: '7' }]) + const [sql, params] = qx.select.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + expect(sql).toMatch(/is_critical/) + expect(sql).toMatch(/daily_downloads_last_run_at/) + expect(params).toMatchObject({ cutoff: '2026-07-15T06:23:00Z', batchSize: 50 }) + }) + + it('bumps the daily watermark with the run result', async () => { + await markPackagistDailyProcessed(asQx(qx), 'pkg:composer/a/x', { + status: 'error', + attempts: 3, + errorKind: 'NOT_FOUND', + }) + + const [sql, params] = qx.result.mock.calls[0] + expect(sql).toMatch(/daily_downloads_last_run_at/) + expect(sql).toMatch(/daily_downloads_run_result/) + expect(sql).toMatch(/ON CONFLICT \(purl\)/) + expect(params).toMatchObject({ purl: 'pkg:composer/a/x' }) + }) +}) + +// Seeding is insert-or-skip: existing rows are never clobbered. +describe('insertPackagistPackages', () => { + it('inserts seed rows with packagist identity and skips existing purls', async () => { + qx.result.mockResolvedValue(2) + + const inserted = await insertPackagistPackages(asQx(qx), [ + { purl: 'pkg:composer/a/x', vendor: 'a', name: 'x' }, + { purl: 'pkg:composer/b/y', vendor: 'b', name: 'y' }, + ]) + + expect(inserted).toBe(2) + const [sql, params] = qx.result.mock.calls[0] + expect(sql).toMatch(/INSERT INTO packages/) + expect(sql).toMatch(/'packagist'/) + expect(sql).toMatch(/'packagist-registry'/) + expect(sql).toMatch(/ON CONFLICT \(purl\) DO NOTHING/) + expect(params).toMatchObject({ + purls: ['pkg:composer/a/x', 'pkg:composer/b/y'], + vendors: ['a', 'b'], + names: ['x', 'y'], + }) + }) + + it('is a no-op for an empty batch', async () => { + expect(await insertPackagistPackages(asQx(qx), [])).toBe(0) + expect(qx.result).not.toHaveBeenCalled() + }) +}) + +// The package-info update targets the packagist row and reports identity for scoping. +describe('updatePackagistPackageStats', () => { + it('returns null when no packagist row matches the purl', async () => { + expect( + await updatePackagistPackageStats(asQx(qx), { + purl: 'pkg:composer/a/x', + description: null, + declaredRepositoryUrl: null, + repositoryUrl: null, + status: 'active', + downloadsLast30d: 1, + totalDownloads: 2, + dependentCount: 3, + }), + ).toBeNull() + const [sql] = qx.selectOneOrNone.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + }) + + it('maps the row to id/isCritical/changedFields', async () => { + qx.selectOneOrNone.mockResolvedValue({ + id: '7', + is_critical: true, + changed_fields: ['packages.description'], + }) + + const result = await updatePackagistPackageStats(asQx(qx), { + purl: 'pkg:composer/a/x', + description: 'd', + declaredRepositoryUrl: 'https://github.com/a/x', + repositoryUrl: 'https://github.com/a/x', + status: 'active', + downloadsLast30d: 1, + totalDownloads: 2, + dependentCount: 3, + }) + + expect(result).toEqual({ + id: '7', + isCritical: true, + changedFields: ['packages.description'], + }) + }) +}) + +// The guard input: how many critical packagist packages exist. +describe('getCriticalPackagistPackageCount', () => { + it('counts critical packagist packages', async () => { + qx.selectOne.mockResolvedValue({ count: '12' }) + + expect(await getCriticalPackagistPackageCount(asQx(qx))).toBe(12) + const [sql] = qx.selectOne.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + expect(sql).toMatch(/is_critical/) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/expandMetadata.test.ts b/services/apps/packages_worker/src/packagist/__tests__/expandMetadata.test.ts new file mode 100644 index 0000000000..b257d41f54 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/expandMetadata.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest' + +import { expandComposerMetadata } from '../expandMetadata' + +// C1 — Composer\MetadataMinifier::expand semantics: the first version object is complete, +// each later object carries only changed keys, '__unset' removes a key. +describe('expandComposerMetadata', () => { + it('expands diffs against the previous version and honors __unset', () => { + const expanded = expandComposerMetadata([ + { + version: '3.0.0', + version_normalized: '3.0.0.0', + time: '2026-01-01T00:00:00+00:00', + license: ['MIT'], + require: { php: '>=8.1', 'psr/log': '^3.0' }, + homepage: 'https://example.org', + }, + { + version: '2.0.0', + version_normalized: '2.0.0.0', + time: '2024-01-01T00:00:00+00:00', + require: { php: '>=7.4', 'psr/log': '^2.0' }, + homepage: '__unset', + }, + { + version: '1.0.0', + version_normalized: '1.0.0.0', + time: '2020-01-01T00:00:00+00:00', + license: ['BSD-3-Clause'], + }, + ]) + + expect(expanded).toEqual([ + { + version: '3.0.0', + version_normalized: '3.0.0.0', + time: '2026-01-01T00:00:00+00:00', + license: ['MIT'], + require: { php: '>=8.1', 'psr/log': '^3.0' }, + homepage: 'https://example.org', + }, + { + // license carried over from 3.0.0, homepage removed via __unset + version: '2.0.0', + version_normalized: '2.0.0.0', + time: '2024-01-01T00:00:00+00:00', + license: ['MIT'], + require: { php: '>=7.4', 'psr/log': '^2.0' }, + }, + { + // require carried over from 2.0.0, license overridden + version: '1.0.0', + version_normalized: '1.0.0.0', + time: '2020-01-01T00:00:00+00:00', + license: ['BSD-3-Clause'], + require: { php: '>=7.4', 'psr/log': '^2.0' }, + }, + ]) + }) + + it('returns independent objects (mutating one expanded version does not leak into others)', () => { + const expanded = expandComposerMetadata([ + { version: '2.0.0', require: { php: '>=8.0' } }, + { version: '1.0.0' }, + ]) + ;(expanded[0] as Record).extra = 'mutated' + expect('extra' in expanded[1]).toBe(false) + }) + + it('returns [] for an empty version list', () => { + expect(expandComposerMetadata([])).toEqual([]) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts b/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts new file mode 100644 index 0000000000..c0dd70d4df --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts @@ -0,0 +1,188 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { buildPackagistUserAgent, fetchPackagistP2, fetchPackagistStats } from '../fetchPackage' + +// Minimal Response stand-in for the global fetch mock. +function fakeResponse( + status: number, + body?: unknown, + opts: { jsonThrows?: boolean; lastModified?: string } = {}, +): Response { + return { + status, + ok: status >= 200 && status < 300, + headers: { + get: (k: string) => + k.toLowerCase() === 'last-modified' ? (opts.lastModified ?? null) : null, + }, + json: async () => { + if (opts.jsonThrows) throw new Error('bad json') + return body + }, + } as unknown as Response +} + +const validStats = { package: { name: 'monolog/monolog', downloads: { monthly: 5 } } } +const validP2 = { + packages: { 'monolog/monolog': [{ version: '2.0.0' }, { version: '1.0.0' }] }, + minified: 'composer/2.0', +} + +afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() + delete process.env.CROWD_PACKAGES_PACKAGIST_MAILTO +}) + +// B1 — polite crawling: UA carries a mailto, configurable via env. +describe('buildPackagistUserAgent', () => { + it('includes a mailto contact by default', () => { + expect(buildPackagistUserAgent()).toMatch(/mailto=/) + }) + + it('uses the env-configured mailto when set', () => { + process.env.CROWD_PACKAGES_PACKAGIST_MAILTO = 'oss@example.org' + expect(buildPackagistUserAgent()).toContain('mailto=oss@example.org') + }) +}) + +// B1 — dynamic (stats) endpoint: status → FetchError kind mapping +describe('fetchPackagistStats', () => { + it('returns the parsed body on 200 with a valid shape, from the dynamic endpoint URL', async () => { + const fetchMock = vi.fn().mockResolvedValue(fakeResponse(200, validStats)) + vi.stubGlobal('fetch', fetchMock) + + expect(await fetchPackagistStats('monolog/monolog')).toEqual(validStats) + expect(fetchMock.mock.calls[0][0]).toBe('https://packagist.org/packages/monolog/monolog.json') + }) + + it('sends a User-Agent containing a mailto', async () => { + const fetchMock = vi.fn().mockResolvedValue(fakeResponse(200, validStats)) + vi.stubGlobal('fetch', fetchMock) + + await fetchPackagistStats('monolog/monolog') + const headers = fetchMock.mock.calls[0][1].headers as Record + expect(headers['User-Agent']).toMatch(/mailto=/) + }) + + it('maps 404 → NOT_FOUND', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(404))) + expect(await fetchPackagistStats('gone/gone')).toMatchObject({ + kind: 'NOT_FOUND', + statusCode: 404, + }) + }) + + it('maps 429 → RATE_LIMIT', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(429))) + expect(await fetchPackagistStats('busy/busy')).toMatchObject({ + kind: 'RATE_LIMIT', + statusCode: 429, + }) + }) + + it('maps other non-ok statuses (5xx) → TRANSIENT with the status code', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(503))) + expect(await fetchPackagistStats('flaky/flaky')).toMatchObject({ + kind: 'TRANSIENT', + statusCode: 503, + }) + }) + + it('maps a network rejection → TRANSIENT without a status code', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('ECONNRESET'))) + const result = await fetchPackagistStats('monolog/monolog') + expect(result).toMatchObject({ kind: 'TRANSIENT' }) + expect((result as { statusCode?: number }).statusCode).toBeUndefined() + }) + + it('maps an unparseable body → MALFORMED', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(fakeResponse(200, undefined, { jsonThrows: true })), + ) + expect(await fetchPackagistStats('monolog/monolog')).toMatchObject({ kind: 'MALFORMED' }) + }) + + it('maps an unexpected JSON shape → MALFORMED', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(200, { not: 'a package' }))) + expect(await fetchPackagistStats('monolog/monolog')).toMatchObject({ kind: 'MALFORMED' }) + }) + + it('maps a body read aborted by the 30s timeout → TRANSIENT (not MALFORMED)', async () => { + vi.useFakeTimers() + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + status: 200, + ok: true, + headers: { get: () => null }, + json: () => + new Promise((_resolve, reject) => setTimeout(() => reject(new Error('aborted')), 40_000)), + } as unknown as Response), + ) + const p = fetchPackagistStats('monolog/monolog') + await vi.advanceTimersByTimeAsync(41_000) + expect(await p).toMatchObject({ kind: 'TRANSIENT' }) + }) +}) + +// B1 + C5 — static p2 endpoint: same error contract plus If-Modified-Since / 304 handling. +describe('fetchPackagistP2', () => { + it('returns the minified versions and Last-Modified on 200, from the static endpoint URL', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue( + fakeResponse(200, validP2, { lastModified: 'Wed, 01 Jul 2026 00:00:00 GMT' }), + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await fetchPackagistP2('monolog/monolog', null) + expect(result).toEqual({ + minifiedVersions: [{ version: '2.0.0' }, { version: '1.0.0' }], + lastModified: 'Wed, 01 Jul 2026 00:00:00 GMT', + }) + expect(fetchMock.mock.calls[0][0]).toBe('https://repo.packagist.org/p2/monolog/monolog.json') + }) + + it('sends If-Modified-Since when a previous Last-Modified is known', async () => { + const fetchMock = vi.fn().mockResolvedValue(fakeResponse(200, validP2)) + vi.stubGlobal('fetch', fetchMock) + + await fetchPackagistP2('monolog/monolog', 'Tue, 30 Jun 2026 00:00:00 GMT') + const headers = fetchMock.mock.calls[0][1].headers as Record + expect(headers['If-Modified-Since']).toBe('Tue, 30 Jun 2026 00:00:00 GMT') + }) + + it('omits If-Modified-Since when none is known', async () => { + const fetchMock = vi.fn().mockResolvedValue(fakeResponse(200, validP2)) + vi.stubGlobal('fetch', fetchMock) + + await fetchPackagistP2('monolog/monolog', null) + const headers = fetchMock.mock.calls[0][1].headers as Record + expect('If-Modified-Since' in headers).toBe(false) + }) + + it('maps 304 → NOT_MODIFIED', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(304))) + expect(await fetchPackagistP2('monolog/monolog', 'Tue, 30 Jun 2026 00:00:00 GMT')).toEqual({ + kind: 'NOT_MODIFIED', + }) + }) + + it('maps 404 → NOT_FOUND and 429 → RATE_LIMIT', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(404))) + expect(await fetchPackagistP2('gone/gone', null)).toMatchObject({ kind: 'NOT_FOUND' }) + + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(429))) + expect(await fetchPackagistP2('busy/busy', null)).toMatchObject({ kind: 'RATE_LIMIT' }) + }) + + it('maps a payload missing the package key → MALFORMED', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(fakeResponse(200, { packages: { 'other/pkg': [] } })), + ) + expect(await fetchPackagistP2('monolog/monolog', null)).toMatchObject({ kind: 'MALFORMED' }) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts new file mode 100644 index 0000000000..94c143404e --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -0,0 +1,432 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { + getPackagistMetadataDuePurls, + insertDailyDownloads, + insertPackagistPackages, + logAuditFieldChanges, + markPackagist30dProcessed, + markPackagistDailyProcessed, + markPackagistMetadataScanned, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { + getPackagistMetadataBatch, + ingestOnePackagist30dWindow, + ingestOnePackagistDailyDownload, + ingestOnePackagistMetadata, + ingestPackagistItemsConcurrently, + runPackagistPackageSeed, +} from '../activities' +import { persistPackagist30dWindow } from '../downloads' +import { expandComposerMetadata } from '../expandMetadata' +import { fetchPackagistP2, fetchPackagistStats } from '../fetchPackage' +import { fetchPackagistPackageList, parsePackagistPackageList } from '../listPackages' +import { INGEST_MAX_ATTEMPTS } from '../retryPolicy' +import { persistPackagistMetadata } from '../upsertMetadata' +import { persistPackagistPackageInfo } from '../upsertPackageInfo' + +// Mock the heavy collaborators so we exercise the classification/retry/give-up logic. +vi.mock('../fetchPackage', () => ({ + fetchPackagistStats: vi.fn(), + fetchPackagistP2: vi.fn(), + buildPackagistUserAgent: vi.fn(() => 'ua'), +})) +vi.mock('../upsertPackageInfo', () => ({ persistPackagistPackageInfo: vi.fn() })) +vi.mock('../upsertMetadata', () => ({ persistPackagistMetadata: vi.fn() })) +vi.mock('../downloads', () => ({ + monthlyWindowFor: vi.fn(), + persistPackagist30dWindow: vi.fn(), +})) +vi.mock('../expandMetadata', () => ({ expandComposerMetadata: vi.fn() })) +vi.mock('../listPackages', () => ({ + fetchPackagistPackageList: vi.fn(), + parsePackagistPackageList: vi.fn(), +})) +vi.mock('@crowd/data-access-layer/src/packages', () => ({ + markPackagistMetadataScanned: vi.fn(), + markPackagist30dProcessed: vi.fn(), + markPackagistDailyProcessed: vi.fn(), + getPackagistMetadataDuePurls: vi.fn(), + getPackagist30dDuePurls: vi.fn(), + getPackagistDailyDownloadsDue: vi.fn(), + getCriticalPackagistPackageCount: vi.fn(), + insertDailyDownloads: vi.fn().mockResolvedValue([]), + insertPackagistPackages: vi.fn(), + logAuditFieldChanges: vi.fn(), +})) + +const mockFetchStats = vi.mocked(fetchPackagistStats) +const mockFetchP2 = vi.mocked(fetchPackagistP2) +const mockExpand = vi.mocked(expandComposerMetadata) +const mockPersistInfo = vi.mocked(persistPackagistPackageInfo) +const mockPersistMetadata = vi.mocked(persistPackagistMetadata) +const mockPersist30d = vi.mocked(persistPackagist30dWindow) +const mockDaily = vi.mocked(insertDailyDownloads) +const mockMarkMetadata = vi.mocked(markPackagistMetadataScanned) +const mockMark30d = vi.mocked(markPackagist30dProcessed) +const mockMarkDaily = vi.mocked(markPackagistDailyProcessed) +const mockAudit = vi.mocked(logAuditFieldChanges) +const mockFetchList = vi.mocked(fetchPackagistPackageList) +const mockParseList = vi.mocked(parsePackagistPackageList) +const mockSeedInsert = vi.mocked(insertPackagistPackages) + +const qx = {} as QueryExecutor +const PURL = 'pkg:composer/monolog/monolog' +const RUN_DATE = '2026-07-15' + +const statsJson = { + package: { name: 'monolog/monolog', downloads: { total: 1000, monthly: 300, daily: 10 } }, +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +afterEach(() => { + vi.useRealTimers() +}) + +// The merged metadata lane: one pass per package fetches BOTH registry endpoints — +// the dynamic one (package info, repos, maintainers) and p2 (versions, dependencies). +describe('ingestOnePackagistMetadata', () => { + const candidate = { purl: PURL, metadataLastModified: 'Tue, 30 Jun 2026 00:00:00 GMT' } + const minified = [{ version: '2.0.0' }] + const expanded = [{ version: '2.0.0', version_normalized: '2.0.0.0' }] + + function happyMocks() { + mockFetchStats.mockResolvedValue(statsJson as never) + mockPersistInfo.mockResolvedValue({ found: true, changedFields: ['packages.description'] }) + mockFetchP2.mockResolvedValue({ + minifiedVersions: minified, + lastModified: 'Wed, 01 Jul 2026 00:00:00 GMT', + } as never) + mockExpand.mockReturnValue(expanded as never) + mockPersistMetadata.mockResolvedValue({ + found: true, + changedFields: ['versions.number'], + unresolvedDependencyTargets: 0, + }) + } + + it('persists both endpoints, audits the merged fields, and marks scanned with the fresh Last-Modified', async () => { + happyMocks() + + await ingestOnePackagistMetadata(qx, candidate) + + expect(mockPersistInfo).toHaveBeenCalledWith(qx, PURL, expect.anything()) + expect(mockFetchP2).toHaveBeenCalledWith('monolog/monolog', 'Tue, 30 Jun 2026 00:00:00 GMT') + expect(mockExpand).toHaveBeenCalledWith(minified) + expect(mockPersistMetadata).toHaveBeenCalledWith(qx, PURL, expanded) + expect(mockAudit).toHaveBeenCalledWith( + qx, + 'packagist', + PURL, + expect.arrayContaining(['packages.description', 'versions.number']), + ) + expect(mockMarkMetadata).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'success', attempts: 1 }), + 'Wed, 01 Jul 2026 00:00:00 GMT', + ) + }) + + it('records a p2 NOT_MODIFIED as success: info persisted, versions skipped', async () => { + happyMocks() + mockFetchP2.mockResolvedValue({ kind: 'NOT_MODIFIED' } as never) + + await ingestOnePackagistMetadata(qx, candidate) + + expect(mockPersistInfo).toHaveBeenCalled() + expect(mockPersistMetadata).not.toHaveBeenCalled() + expect(mockMarkMetadata).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'success' }), + ) + }) + + it('gives up on a persistent dynamic 404 without touching p2', async () => { + vi.useFakeTimers() + mockFetchStats.mockResolvedValue({ + kind: 'NOT_FOUND', + statusCode: 404, + message: 'not found', + } as never) + + const p = ingestOnePackagistMetadata(qx, candidate) + await vi.runAllTimersAsync() + await p + + expect(mockFetchStats).toHaveBeenCalledTimes(3) + expect(mockFetchP2).not.toHaveBeenCalled() + expect(mockPersistInfo).not.toHaveBeenCalled() + expect(mockMarkMetadata).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND', httpStatus: 404 }), + ) + }) + + it('throws on a transient dynamic result without marking scanned', async () => { + mockFetchStats.mockResolvedValue({ + kind: 'TRANSIENT', + statusCode: 500, + message: 'HTTP 500', + } as never) + + await expect(ingestOnePackagistMetadata(qx, candidate)).rejects.toThrow() + expect(mockMarkMetadata).not.toHaveBeenCalled() + expect(mockFetchStats).toHaveBeenCalledTimes(1) + }) + + it('gives up on a persistent p2 404 after fast retries and marks it scanned(error)', async () => { + vi.useFakeTimers() + mockFetchStats.mockResolvedValue(statsJson as never) + mockPersistInfo.mockResolvedValue({ found: true, changedFields: [] }) + mockFetchP2.mockResolvedValue({ + kind: 'NOT_FOUND', + statusCode: 404, + message: 'not found', + } as never) + + const p = ingestOnePackagistMetadata(qx, candidate) + await vi.runAllTimersAsync() + await p + + expect(mockPersistMetadata).not.toHaveBeenCalled() + expect(mockMarkMetadata).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), + ) + }) + + it('audits phase-1 changes even when the p2 fetch gives up', async () => { + // the dynamic-endpoint writes are committed by then — their audit rows must not be dropped + vi.useFakeTimers() + mockFetchStats.mockResolvedValue(statsJson as never) + mockPersistInfo.mockResolvedValue({ found: true, changedFields: ['packages.description'] }) + mockFetchP2.mockResolvedValue({ + kind: 'NOT_FOUND', + statusCode: 404, + message: 'not found', + } as never) + + const p = ingestOnePackagistMetadata(qx, candidate) + await vi.runAllTimersAsync() + await p + + expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, ['packages.description']) + expect(mockMarkMetadata).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), + ) + }) + + it('throws on a transient p2 result without marking scanned', async () => { + mockFetchStats.mockResolvedValue(statsJson as never) + mockPersistInfo.mockResolvedValue({ found: true, changedFields: [] }) + mockFetchP2.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 502' } as never) + + await expect(ingestOnePackagistMetadata(qx, candidate)).rejects.toThrow() + expect(mockMarkMetadata).not.toHaveBeenCalled() + }) +}) + +// The monthly downloads-30d lane: dynamic fetch, window row only. +describe('ingestOnePackagist30dWindow', () => { + it('persists the observed rolling window and marks the run processed', async () => { + mockFetchStats.mockResolvedValue(statsJson as never) + + await ingestOnePackagist30dWindow(qx, PURL, RUN_DATE) + + expect(mockPersist30d).toHaveBeenCalledWith(qx, PURL, 300, RUN_DATE) + expect(mockMark30d).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'success', attempts: 1 }), + ) + }) + + it('gives up on a persistent 404 and marks the run errored', async () => { + vi.useFakeTimers() + mockFetchStats.mockResolvedValue({ + kind: 'NOT_FOUND', + statusCode: 404, + message: 'not found', + } as never) + + const p = ingestOnePackagist30dWindow(qx, PURL, RUN_DATE) + await vi.runAllTimersAsync() + await p + + expect(mockPersist30d).not.toHaveBeenCalled() + expect(mockMark30d).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), + ) + }) + + it('throws on a transient result without marking processed', async () => { + mockFetchStats.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 503' } as never) + + await expect(ingestOnePackagist30dWindow(qx, PURL, RUN_DATE)).rejects.toThrow() + expect(mockMark30d).not.toHaveBeenCalled() + }) +}) + +// The daily downloads lane (critical slice): dynamic fetch, one downloads_daily row. +describe('ingestOnePackagistDailyDownload', () => { + const candidate = { purl: PURL, packageId: '7' } + + it('inserts the daily row and marks the run processed', async () => { + mockFetchStats.mockResolvedValue(statsJson as never) + + await ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE) + + expect(mockDaily).toHaveBeenCalledWith(qx, '7', [{ day: RUN_DATE, downloads: 10 }]) + expect(mockMarkDaily).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'success' }), + ) + }) + + it('marks success without inserting when the registry reports no daily count', async () => { + mockFetchStats.mockResolvedValue({ package: { name: 'monolog/monolog' } } as never) + + await ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE) + + expect(mockDaily).not.toHaveBeenCalled() + expect(mockMarkDaily).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'success' }), + ) + }) + + it('throws on a transient result without marking processed', async () => { + mockFetchStats.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 503' } as never) + + await expect(ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE)).rejects.toThrow() + expect(mockMarkDaily).not.toHaveBeenCalled() + }) +}) + +// Concurrency-bounded batch processing with the pypi give-up contract: transient failures +// rethrow while Temporal retries remain, then are given up per-item so the cursor advances. +describe('ingestPackagistItemsConcurrently', () => { + const items = ['a', 'bad', 'c', 'd', 'e', 'f'] + const makeIngest = () => + vi.fn((item: string) => + item === 'bad' ? Promise.reject(new Error('transient')) : Promise.resolve(), + ) + + it('rethrows while Temporal retries remain and does not give up on any item', async () => { + const ingest = makeIngest() + const onGiveUp = vi.fn().mockResolvedValue(undefined) + + await expect(ingestPackagistItemsConcurrently(items, 1, 2, ingest, onGiveUp)).rejects.toThrow() + expect(onGiveUp).not.toHaveBeenCalled() + }) + + it('on the final attempt gives up on the failing item and completes the rest', async () => { + const ingest = makeIngest() + const onGiveUp = vi.fn().mockResolvedValue(undefined) + + await expect( + ingestPackagistItemsConcurrently(items, INGEST_MAX_ATTEMPTS, 2, ingest, onGiveUp), + ).resolves.toBeUndefined() + + expect(ingest).toHaveBeenCalledTimes(6) + expect(onGiveUp).toHaveBeenCalledTimes(1) + expect(onGiveUp.mock.calls[0][0]).toBe('bad') + }) + + it('never runs more than `concurrency` ingests at once', async () => { + let inFlight = 0 + let maxInFlight = 0 + const ingest = vi.fn(async () => { + inFlight += 1 + maxInFlight = Math.max(maxInFlight, inFlight) + await new Promise((resolve) => setTimeout(resolve, 5)) + inFlight -= 1 + }) + + await ingestPackagistItemsConcurrently(items, 1, 2, ingest, vi.fn()) + + expect(ingest).toHaveBeenCalledTimes(6) + expect(maxInFlight).toBeLessThanOrEqual(2) + }) +}) + +// All-packages metadata scope: the env default drives due-selection's onlyCritical arg. +// Default is ALL packages (deps.dev has no Packagist data to fall back on); the env var +// narrows back to the critical slice. +describe('getPackagistMetadataBatch scope', () => { + const mockMetadataDue = vi.mocked(getPackagistMetadataDuePurls) + + afterEach(() => { + delete process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL + }) + + it('selects across ALL packagist packages by default', async () => { + mockMetadataDue.mockResolvedValue([]) + + await getPackagistMetadataBatch('', 50) + + expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), '', 50, 7, false) + }) + + it('narrows to the critical slice when CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL=true', async () => { + process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL = 'true' + mockMetadataDue.mockResolvedValue([]) + + await getPackagistMetadataBatch('', 50) + + expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), '', 50, 7, true) + }) + + it('keeps the all-packages default when the env value is unrecognized', async () => { + // a typo must not silently flip the sweep to critical-only + process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL = 'ture' + mockMetadataDue.mockResolvedValue([]) + + await getPackagistMetadataBatch('', 50) + + expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), '', 50, 7, false) + }) +}) + +// A1/A2 — the seed activity: one list fetch, parsed entries inserted, counts reported. +describe('runPackagistPackageSeed', () => { + it('fetches the list once, seeds all parsed entries, and reports counts', async () => { + const entries = [ + { vendor: 'a', name: 'x', purl: 'pkg:composer/a/x' }, + { vendor: 'b', name: 'y', purl: 'pkg:composer/b/y' }, + { vendor: 'c', name: 'z', purl: 'pkg:composer/c/z' }, + ] + mockFetchList.mockResolvedValue({ packageNames: ['raw'] }) + mockParseList.mockReturnValue({ entries, invalid: 2 }) + mockSeedInsert.mockResolvedValue(3) + + const result = await runPackagistPackageSeed() + + expect(mockFetchList).toHaveBeenCalledTimes(1) + const seeded = mockSeedInsert.mock.calls.flatMap((c) => c[1]) + expect(seeded).toEqual(entries) + expect(result).toEqual({ discovered: 3, invalid: 2 }) + }) + + it('throws when the list fetch fails so Temporal retries the seed', async () => { + mockFetchList.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 502' }) + + await expect(runPackagistPackageSeed()).rejects.toThrow() + expect(mockSeedInsert).not.toHaveBeenCalled() + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts b/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts new file mode 100644 index 0000000000..31d39ebab0 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts @@ -0,0 +1,88 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { fetchPackagistPackageList, parsePackagistPackageList } from '../listPackages' + +afterEach(() => { + vi.unstubAllGlobals() +}) + +// A1 — list.json parsing: vendor/name split, purl form, invalid skipped+counted, dedup. +describe('parsePackagistPackageList', () => { + it('maps names to vendor/name entries with pkg:composer purls', () => { + const { entries, invalid } = parsePackagistPackageList({ + packageNames: ['monolog/monolog', 'symfony/http-kernel'], + }) + expect(entries).toEqual([ + { vendor: 'monolog', name: 'monolog', purl: 'pkg:composer/monolog/monolog' }, + { vendor: 'symfony', name: 'http-kernel', purl: 'pkg:composer/symfony/http-kernel' }, + ]) + expect(invalid).toBe(0) + }) + + it('accepts dots, underscores and hyphens in segments', () => { + const { entries } = parsePackagistPackageList({ + packageNames: ['foo/bar.baz', 'a1/b_c', 'x-y/z-1'], + }) + expect(entries.map((e) => e.purl)).toEqual([ + 'pkg:composer/foo/bar.baz', + 'pkg:composer/a1/b_c', + 'pkg:composer/x-y/z-1', + ]) + }) + + it('skips and counts invalid names', () => { + const { entries, invalid } = parsePackagistPackageList({ + packageNames: [ + 'valid/name', + 'noslash', + 'too/many/parts', + '/leading', + 'trailing/', + 'has space/pkg', + '', + 42, + ], + }) + expect(entries.map((e) => e.purl)).toEqual(['pkg:composer/valid/name']) + expect(invalid).toBe(7) + }) + + it('lowercases and deduplicates names without counting duplicates as invalid', () => { + const { entries, invalid } = parsePackagistPackageList({ + packageNames: ['Monolog/Monolog', 'monolog/monolog'], + }) + expect(entries).toEqual([ + { vendor: 'monolog', name: 'monolog', purl: 'pkg:composer/monolog/monolog' }, + ]) + expect(invalid).toBe(0) + }) + + it('throws on a root shape without a packageNames array', () => { + expect(() => parsePackagistPackageList({ nope: true })).toThrow() + expect(() => parsePackagistPackageList({ packageNames: 'not-an-array' })).toThrow() + expect(() => parsePackagistPackageList(null)).toThrow() + }) +}) + +describe('fetchPackagistPackageList', () => { + it('returns the raw JSON on 200 and hits the list endpoint', async () => { + const body = { packageNames: ['monolog/monolog'] } + const fetchMock = vi.fn().mockResolvedValue({ + status: 200, + ok: true, + json: async () => body, + } as unknown as Response) + vi.stubGlobal('fetch', fetchMock) + + expect(await fetchPackagistPackageList()).toEqual(body) + expect(fetchMock.mock.calls[0][0]).toBe('https://packagist.org/packages/list.json') + }) + + it('maps a 5xx to TRANSIENT', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ status: 503, ok: false, json: async () => ({}) }), + ) + expect(await fetchPackagistPackageList()).toMatchObject({ kind: 'TRANSIENT', statusCode: 503 }) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts new file mode 100644 index 0000000000..511a8916cf --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts @@ -0,0 +1,260 @@ +import { describe, expect, it } from 'vitest' + +import { + buildPackagistVersionRows, + extractVersionDependencies, + isPackagistDevVersion, + isPackagistPrerelease, + normalizePackagistStats, + packagistNameFromPurl, + purlFromPackagistName, +} from '../normalize' +import type { PackagistPackageInfo } from '../types' + +describe('purl helpers', () => { + it('round-trips vendor/name through pkg:composer purls', () => { + expect(purlFromPackagistName('monolog/monolog')).toBe('pkg:composer/monolog/monolog') + expect(packagistNameFromPurl('pkg:composer/monolog/monolog')).toBe('monolog/monolog') + expect(packagistNameFromPurl('pkg:composer/symfony/http-kernel')).toBe('symfony/http-kernel') + }) +}) + +// B2 — dynamic-endpoint normalization. +describe('normalizePackagistStats', () => { + const full: PackagistPackageInfo = { + name: 'monolog/monolog', + description: 'Sends your logs to files and web services', + time: '2011-02-15T15:11:52+00:00', + maintainers: [{ name: 'seldaek', avatar_url: 'https://example.org/a.png' }, {}], + type: 'library', + repository: 'https://github.com/Seldaek/monolog', + language: 'PHP', + dependents: 5423, + suggesters: 300, + favers: 10000, + downloads: { total: 500_000_000, monthly: 10_000_000, daily: 350_000 }, + } + + it('extracts description, repository, dependents and the three download counters', () => { + const stats = normalizePackagistStats(full) + expect(stats).toEqual({ + name: 'monolog/monolog', + description: 'Sends your logs to files and web services', + repositoryUrl: 'https://github.com/Seldaek/monolog', + status: 'active', + dependents: 5423, + downloadsTotal: 500_000_000, + downloadsMonthly: 10_000_000, + // nameless maintainer entries are dropped + maintainers: [{ username: 'seldaek', displayName: null, email: null, role: 'maintainer' }], + }) + }) + + it('maps abandoned → deprecated for both the boolean and replacement-name forms', () => { + expect(normalizePackagistStats({ ...full, abandoned: true }).status).toBe('deprecated') + expect(normalizePackagistStats({ ...full, abandoned: 'new/package' }).status).toBe('deprecated') + expect(normalizePackagistStats({ ...full, abandoned: false }).status).toBe('active') + expect(normalizePackagistStats(full).status).toBe('active') + }) + + it('tolerates missing optional fields with nulls and empty lists', () => { + const stats = normalizePackagistStats({ name: 'a/b' }) + expect(stats).toEqual({ + name: 'a/b', + description: null, + repositoryUrl: null, + status: 'active', + dependents: null, + downloadsTotal: null, + downloadsMonthly: null, + maintainers: [], + }) + }) +}) + +describe('isPackagistDevVersion', () => { + it('detects dev- branch versions', () => { + expect(isPackagistDevVersion('dev-main')).toBe(true) + expect(isPackagistDevVersion('dev-feature/foo')).toBe(true) + }) + + it('detects numbered branch versions via the -dev normalized suffix', () => { + expect(isPackagistDevVersion('2.x-dev', '2.9999999.9999999.9999999-dev')).toBe(true) + expect(isPackagistDevVersion('1.0.x-dev', '1.0.9999999.9999999-dev')).toBe(true) + }) + + it('keeps tagged releases', () => { + expect(isPackagistDevVersion('1.0.0', '1.0.0.0')).toBe(false) + expect(isPackagistDevVersion('v2.3.4', '2.3.4.0')).toBe(false) + }) +}) + +describe('isPackagistPrerelease', () => { + it('flags alpha/beta/RC suffixes', () => { + expect(isPackagistPrerelease('1.0.0.0-alpha2')).toBe(true) + expect(isPackagistPrerelease('1.0.0.0-beta1')).toBe(true) + expect(isPackagistPrerelease('1.0.0.0-RC1')).toBe(true) + }) + + it('treats plain and patch-suffixed versions as stable', () => { + expect(isPackagistPrerelease('1.0.0.0')).toBe(false) + expect(isPackagistPrerelease('1.0.0.0-patch1')).toBe(false) + }) +}) + +// C2 — version rows: dev branches skipped, latest by normalized numeric compare, +// stable preferred over prerelease, aggregates span the kept rows. +describe('buildPackagistVersionRows', () => { + it('skips dev branches and derives rows and aggregates from tagged releases', () => { + const { versionRows, latestVersion, firstReleaseAt, latestReleaseAt, licenses, homepage } = + buildPackagistVersionRows([ + { version: 'dev-main', version_normalized: 'dev-main' }, + { + version: '2.10.0', + version_normalized: '2.10.0.0', + time: '2024-03-01T00:00:00+00:00', + license: ['MIT'], + homepage: 'https://example.org', + }, + { + version: '2.9.1', + version_normalized: '2.9.1.0', + time: '2023-01-01T00:00:00+00:00', + license: ['MIT'], + }, + { + version: '1.0.0', + version_normalized: '1.0.0.0', + time: '2020-06-01T00:00:00+00:00', + license: ['BSD-3-Clause'], + homepage: 'https://old.example.org', + }, + ]) + + expect(versionRows).toHaveLength(3) + const byNumber = Object.fromEntries(versionRows.map((r) => [r.number, r])) + // numeric compare: 2.10.0 > 2.9.1 + expect(byNumber['2.10.0']).toEqual({ + number: '2.10.0', + publishedAt: '2024-03-01T00:00:00+00:00', + isLatest: true, + isPrerelease: false, + license: 'MIT', + }) + expect(byNumber['2.9.1'].isLatest).toBe(false) + expect(byNumber['1.0.0'].license).toBe('BSD-3-Clause') + expect(latestVersion).toBe('2.10.0') + expect(firstReleaseAt).toBe('2020-06-01T00:00:00+00:00') + expect(latestReleaseAt).toBe('2024-03-01T00:00:00+00:00') + // package-level licenses and homepage come from the latest version + expect(licenses).toEqual(['MIT']) + expect(homepage).toBe('https://example.org') + }) + + it('returns a null homepage when the latest version omits or blanks it', () => { + const absent = buildPackagistVersionRows([{ version: '1.0.0', version_normalized: '1.0.0.0' }]) + expect(absent.homepage).toBeNull() + + const blank = buildPackagistVersionRows([ + { version: '1.0.0', version_normalized: '1.0.0.0', homepage: ' ' }, + ]) + expect(blank.homepage).toBeNull() + }) + + it('prefers a stable release over a newer prerelease for latest', () => { + const { latestVersion, versionRows } = buildPackagistVersionRows([ + { + version: '3.0.0-RC1', + version_normalized: '3.0.0.0-RC1', + time: '2026-01-01T00:00:00+00:00', + }, + { version: '2.5.0', version_normalized: '2.5.0.0', time: '2025-01-01T00:00:00+00:00' }, + ]) + expect(latestVersion).toBe('2.5.0') + const byNumber = Object.fromEntries(versionRows.map((r) => [r.number, r])) + expect(byNumber['3.0.0-RC1'].isPrerelease).toBe(true) + expect(byNumber['3.0.0-RC1'].isLatest).toBe(false) + expect(byNumber['2.5.0'].isLatest).toBe(true) + }) + + it('falls back to the highest prerelease when no stable release exists', () => { + const { latestVersion } = buildPackagistVersionRows([ + { version: '1.0.0-beta2', version_normalized: '1.0.0.0-beta2' }, + { version: '1.0.0-alpha3', version_normalized: '1.0.0.0-alpha3' }, + ]) + expect(latestVersion).toBe('1.0.0-beta2') + }) + + it('picks the higher base version among prereleases regardless of suffix rank', () => { + const { latestVersion } = buildPackagistVersionRows([ + { version: '1.0.0-rc1', version_normalized: '1.0.0.0-rc1' }, + { version: '2.0.0-alpha1', version_normalized: '2.0.0.0-alpha1' }, + ]) + expect(latestVersion).toBe('2.0.0-alpha1') + }) + + it('breaks same-base prerelease ties by suffix rank, not array order', () => { + // lower-precedence suffix listed FIRST — array order must not decide + const { latestVersion } = buildPackagistVersionRows([ + { version: '1.0.0-alpha3', version_normalized: '1.0.0.0-alpha3' }, + { version: '1.0.0-beta2', version_normalized: '1.0.0.0-beta2' }, + ]) + expect(latestVersion).toBe('1.0.0-beta2') + }) + + it('distinguishes numbered suffixes of the same rank', () => { + const { latestVersion } = buildPackagistVersionRows([ + { version: '1.0.0-rc1', version_normalized: '1.0.0.0-rc1' }, + { version: '1.0.0-rc2', version_normalized: '1.0.0.0-rc2' }, + ]) + expect(latestVersion).toBe('1.0.0-rc2') + }) + + it('returns empty rows and null aggregates when only dev branches exist', () => { + expect( + buildPackagistVersionRows([{ version: 'dev-main', version_normalized: 'dev-main' }]), + ).toEqual({ + versionRows: [], + latestVersion: null, + firstReleaseAt: null, + latestReleaseAt: null, + licenses: null, + homepage: null, + }) + }) +}) + +// C3 — dependency extraction: require→direct, require-dev→dev, platform packages excluded. +describe('extractVersionDependencies', () => { + it('maps require to direct and require-dev to dev, preserving constraints', () => { + const deps = extractVersionDependencies({ + version: '2.0.0', + require: { php: '>=8.1', 'psr/log': '^2.0 || ^3.0' }, + 'require-dev': { 'phpunit/phpunit': '^10.5' }, + }) + expect(deps).toEqual([ + { name: 'psr/log', constraint: '^2.0 || ^3.0', kind: 'direct' }, + { name: 'phpunit/phpunit', constraint: '^10.5', kind: 'dev' }, + ]) + }) + + it('excludes platform packages (php, hhvm, composer-*, ext-*, lib-*)', () => { + const deps = extractVersionDependencies({ + version: '1.0.0', + require: { + php: '>=7.4', + hhvm: '*', + 'composer-plugin-api': '^2.0', + 'composer-runtime-api': '^2.2', + 'ext-json': '*', + 'lib-icu': '*', + 'real/dep': '^1.0', + }, + }) + expect(deps).toEqual([{ name: 'real/dep', constraint: '^1.0', kind: 'direct' }]) + }) + + it('returns [] when the version declares no dependencies', () => { + expect(extractVersionDependencies({ version: '1.0.0' })).toEqual([]) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts new file mode 100644 index 0000000000..df9dbc02ba --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts @@ -0,0 +1,165 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + getPackagistPackageIdsByNames, + updatePackagistVersionAggregates, + upsertPackagistVersions, + upsertVersionDependencies, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import type { PackagistExpandedVersion } from '../types' +import { persistPackagistMetadata } from '../upsertMetadata' + +vi.mock('@crowd/data-access-layer/src/packages', () => ({ + updatePackagistVersionAggregates: vi.fn(), + upsertPackagistVersions: vi.fn(), + getPackagistPackageIdsByNames: vi.fn(), + upsertVersionDependencies: vi.fn().mockResolvedValue(0), +})) + +const mockAggregates = vi.mocked(updatePackagistVersionAggregates) +const mockVersions = vi.mocked(upsertPackagistVersions) +const mockIds = vi.mocked(getPackagistPackageIdsByNames) +const mockDeps = vi.mocked(upsertVersionDependencies) + +const qx = {} as QueryExecutor +const PURL = 'pkg:composer/monolog/monolog' + +const expanded: PackagistExpandedVersion[] = [ + { + version: '2.0.0', + version_normalized: '2.0.0.0', + time: '2024-01-01T00:00:00+00:00', + license: ['MIT'], + homepage: 'https://monolog.example.org', + require: { php: '>=8.1', 'psr/log': '^3.0' }, + 'require-dev': { 'phpunit/phpunit': '^10.5' }, + }, + { + version: '1.0.0', + version_normalized: '1.0.0.0', + time: '2020-01-01T00:00:00+00:00', + license: ['MIT'], + require: { 'psr/log': '^1.0' }, + }, + { version: 'dev-main', version_normalized: 'dev-main' }, +] + +beforeEach(() => { + vi.clearAllMocks() + mockDeps.mockResolvedValue(0) +}) + +// C2 + C3 — persistence wiring: aggregates on packages, version rows, dependency edges +// resolved to existing packages rows only. +describe('persistPackagistMetadata', () => { + it('upserts version rows (dev branches excluded), aggregates, and resolved dependency edges', async () => { + mockAggregates.mockResolvedValue({ id: '9', changedFields: ['packages.latest_version'] }) + mockVersions.mockResolvedValue({ + changedFields: ['versions.number'], + versionIds: [ + { number: '2.0.0', id: '21' }, + { number: '1.0.0', id: '20' }, + ], + }) + mockIds.mockResolvedValue( + new Map([ + ['psr/log', '44'], + ['phpunit/phpunit', '45'], + ]), + ) + + const result = await persistPackagistMetadata(qx, PURL, expanded) + + expect(mockAggregates).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ + versionsCount: 2, + latestVersion: '2.0.0', + firstReleaseAt: '2020-01-01T00:00:00+00:00', + latestReleaseAt: '2024-01-01T00:00:00+00:00', + licenses: ['MIT'], + homepage: 'https://monolog.example.org', + }), + ) + expect(mockVersions).toHaveBeenCalledWith( + qx, + '9', + expect.arrayContaining([ + expect.objectContaining({ number: '2.0.0', isLatest: true }), + expect.objectContaining({ number: '1.0.0', isLatest: false }), + ]), + '2.0.0', + ) + expect(mockVersions.mock.calls[0][2]).toHaveLength(2) + + // platform 'php' never reaches resolution + expect(mockIds).toHaveBeenCalledTimes(1) + const requestedNames = mockIds.mock.calls[0][1] + expect([...requestedNames].sort()).toEqual(['phpunit/phpunit', 'psr/log']) + + expect(mockDeps).toHaveBeenCalledTimes(1) + const edges = mockDeps.mock.calls[0][1] + expect(edges).toEqual( + expect.arrayContaining([ + { packageId: '9', versionId: '21', dependsOnId: '44', constraint: '^3.0', kind: 'direct' }, + { packageId: '9', versionId: '21', dependsOnId: '45', constraint: '^10.5', kind: 'dev' }, + { packageId: '9', versionId: '20', dependsOnId: '44', constraint: '^1.0', kind: 'direct' }, + ]), + ) + expect(edges).toHaveLength(3) + + expect(result.found).toBe(true) + expect(result.unresolvedDependencyTargets).toBe(0) + }) + + it('skips and counts dependency targets that do not resolve to a packages row', async () => { + mockAggregates.mockResolvedValue({ id: '9', changedFields: [] }) + mockVersions.mockResolvedValue({ + changedFields: [], + versionIds: [ + { number: '2.0.0', id: '21' }, + { number: '1.0.0', id: '20' }, + ], + }) + // phpunit/phpunit is unknown + mockIds.mockResolvedValue(new Map([['psr/log', '44']])) + + const result = await persistPackagistMetadata(qx, PURL, expanded) + + const edges = mockDeps.mock.calls[0][1] + expect(edges).toHaveLength(2) + expect(edges.every((e) => e.dependsOnId === '44')).toBe(true) + expect(result.unresolvedDependencyTargets).toBe(1) + }) + + it('returns found=false and writes nothing when the packages row is missing', async () => { + mockAggregates.mockResolvedValue(null) + + const result = await persistPackagistMetadata(qx, PURL, expanded) + + expect(result).toEqual({ found: false, changedFields: [], unresolvedDependencyTargets: 0 }) + expect(mockVersions).not.toHaveBeenCalled() + expect(mockIds).not.toHaveBeenCalled() + expect(mockDeps).not.toHaveBeenCalled() + }) + + it('handles a dev-branches-only package: aggregates written, no version or dependency writes', async () => { + mockAggregates.mockResolvedValue({ id: '9', changedFields: [] }) + + const result = await persistPackagistMetadata(qx, PURL, [ + { version: 'dev-main', version_normalized: 'dev-main' }, + ]) + + expect(mockAggregates).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ versionsCount: 0, latestVersion: null, homepage: null }), + ) + expect(mockVersions).not.toHaveBeenCalled() + expect(mockDeps).not.toHaveBeenCalled() + expect(result.found).toBe(true) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts new file mode 100644 index 0000000000..0d90227b7c --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts @@ -0,0 +1,135 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + getOrCreateRepoByUrl, + updatePackagistPackageStats, + upsertPackageMaintainers, + upsertPackageRepo, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import type { NormalizedPackagistStats } from '../types' +import { persistPackagistPackageInfo } from '../upsertPackageInfo' + +vi.mock('@crowd/data-access-layer/src/packages', () => ({ + updatePackagistPackageStats: vi.fn(), + upsertPackageMaintainers: vi.fn().mockResolvedValue([]), + getOrCreateRepoByUrl: vi.fn(), + upsertPackageRepo: vi.fn().mockResolvedValue([]), +})) + +const mockUpdate = vi.mocked(updatePackagistPackageStats) +const mockMaintainers = vi.mocked(upsertPackageMaintainers) +const mockRepoGet = vi.mocked(getOrCreateRepoByUrl) +const mockRepoLink = vi.mocked(upsertPackageRepo) + +const qx = {} as QueryExecutor +const PURL = 'pkg:composer/monolog/monolog' + +const stats: NormalizedPackagistStats = { + name: 'monolog/monolog', + description: 'logs', + repositoryUrl: 'https://github.com/Seldaek/monolog', + status: 'active', + dependents: 42, + downloadsTotal: 1000, + downloadsMonthly: 300, + maintainers: [{ username: 'seldaek', displayName: null, email: null, role: 'maintainer' }], +} + +beforeEach(() => { + vi.clearAllMocks() + mockRepoGet.mockResolvedValue({ id: '55', changedFields: [] }) +}) + +// Dynamic-endpoint persistence inside the metadata lane: packages fields + repo link for +// ALL packages; maintainers only for critical ones. Download rows are NOT written here — +// they belong to the dedicated downloads-30d/daily lanes. +describe('persistPackagistPackageInfo', () => { + it('updates the packages row, links the repo, and writes maintainers for a critical package', async () => { + mockUpdate.mockResolvedValue({ + id: '7', + isCritical: true, + changedFields: ['packages.description'], + }) + + const result = await persistPackagistPackageInfo(qx, PURL, stats) + + expect(mockUpdate).toHaveBeenCalledWith( + qx, + expect.objectContaining({ + purl: PURL, + description: 'logs', + status: 'active', + downloadsLast30d: 300, + totalDownloads: 1000, + dependentCount: 42, + }), + ) + // canonicalized (lowercased) url + coarse host, linked with the manifest-declared convention + expect(mockRepoGet).toHaveBeenCalledWith(qx, 'https://github.com/seldaek/monolog', 'github') + expect(mockRepoLink).toHaveBeenCalledWith(qx, '7', '55', 'declared', 0.8) + expect(mockMaintainers).toHaveBeenCalledWith(qx, '7', stats.maintainers, 'packagist') + expect(result.found).toBe(true) + expect(result.changedFields).toContain('packages.description') + }) + + it('skips maintainers for a non-critical package but still links the repo', async () => { + mockUpdate.mockResolvedValue({ id: '8', isCritical: false, changedFields: [] }) + + await persistPackagistPackageInfo(qx, PURL, stats) + + expect(mockMaintainers).not.toHaveBeenCalled() + expect(mockRepoGet).toHaveBeenCalledWith(qx, 'https://github.com/seldaek/monolog', 'github') + expect(mockRepoLink).toHaveBeenCalledWith(qx, '8', '55', 'declared', 0.8) + }) + + it('skips the repo link when the package declares no repository URL', async () => { + mockUpdate.mockResolvedValue({ id: '7', isCritical: true, changedFields: [] }) + + await persistPackagistPackageInfo(qx, PURL, { ...stats, repositoryUrl: null }) + + expect(mockRepoGet).not.toHaveBeenCalled() + expect(mockRepoLink).not.toHaveBeenCalled() + }) + + it('skips the repo link when the repository URL cannot be canonicalized', async () => { + mockUpdate.mockResolvedValue({ id: '7', isCritical: true, changedFields: [] }) + + await persistPackagistPackageInfo(qx, PURL, { ...stats, repositoryUrl: 'not-a-valid-url' }) + + expect(mockRepoGet).not.toHaveBeenCalled() + expect(mockRepoLink).not.toHaveBeenCalled() + }) + + it('folds repo changed-fields into the result for the audit log', async () => { + mockUpdate.mockResolvedValue({ + id: '7', + isCritical: false, + changedFields: ['packages.description'], + }) + mockRepoGet.mockResolvedValue({ id: '55', changedFields: ['repos.url', 'repos.host'] }) + mockRepoLink.mockResolvedValue(['package_repos.repo_id']) + + const result = await persistPackagistPackageInfo(qx, PURL, stats) + + expect(result.changedFields).toEqual( + expect.arrayContaining([ + 'packages.description', + 'repos.url', + 'repos.host', + 'package_repos.repo_id', + ]), + ) + }) + + it('returns found=false and writes nothing else when the packages row is missing', async () => { + mockUpdate.mockResolvedValue(null) + + const result = await persistPackagistPackageInfo(qx, PURL, stats) + + expect(result).toEqual({ found: false, changedFields: [] }) + expect(mockRepoGet).not.toHaveBeenCalled() + expect(mockMaintainers).not.toHaveBeenCalled() + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/wiring.test.ts b/services/apps/packages_worker/src/packagist/__tests__/wiring.test.ts new file mode 100644 index 0000000000..dda12c6401 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/wiring.test.ts @@ -0,0 +1,49 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' + +import { FETCHERS } from '../../security-contacts/extractors/registry/index' +import { PACKAGIST_CRONS } from '../schedule' + +// D1 — wiring that unit tests can meaningfully pin down. + +describe('package.json worker scripts', () => { + // vitest runs with cwd at the package root; import.meta is unavailable under the CJS tsconfig. + const pkg = JSON.parse(readFileSync('package.json', 'utf8')) as { + scripts: Record + } + + it('declares start/dev scripts for the packagist worker', () => { + expect(pkg.scripts['start:packagist-worker']).toContain('src/bin/packagist-worker.ts') + expect(pkg.scripts['dev:packagist-worker']).toContain('src/bin/packagist-worker.ts') + }) +}) + +describe('schedule cadence', () => { + it('defines the three packagist crons with minutes off :00 (crawler guideline)', () => { + // metadata has no cron — the seed workflow chains it as a child on completion + const crons = Object.entries(PACKAGIST_CRONS) + expect(crons.map(([name]) => name).sort()).toEqual(['downloads30d', 'downloadsDaily', 'seed']) + for (const [name, cron] of crons) { + const minute = cron.split(' ')[0] + expect(minute, `${name} cron minute`).toMatch(/^[1-9][0-9]?$/) + expect(Number(minute), `${name} cron minute`).toBeLessThan(60) + } + }) + + it('runs seed weekly, the 30d window capture monthly on the 1st, and daily downloads daily', () => { + expect(PACKAGIST_CRONS.seed.split(' ')).toHaveLength(5) + expect(PACKAGIST_CRONS.seed.split(' ')[4]).not.toBe('*') + // monthly, anchored on the 1st so the observed rolling value sits on the boundary + expect(PACKAGIST_CRONS.downloads30d.split(' ')[2]).toBe('1') + expect(PACKAGIST_CRONS.downloads30d.split(' ')[4]).toBe('*') + expect(PACKAGIST_CRONS.downloadsDaily.split(' ')[2]).toBe('*') + expect(PACKAGIST_CRONS.downloadsDaily.split(' ')[4]).toBe('*') + }) +}) + +describe('security-contacts registry fetchers', () => { + it('keys the Composer fetcher by the packagist ecosystem value', () => { + expect(FETCHERS.packagist).toBeTypeOf('function') + expect('composer' in FETCHERS).toBe(false) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts new file mode 100644 index 0000000000..7499f65a63 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -0,0 +1,392 @@ +import { Context } from '@temporalio/activity' + +import { partition, timeout } from '@crowd/common' +import { + getCriticalPackagistPackageCount, + getPackagist30dDuePurls, + getPackagistDailyDownloadsDue, + getPackagistMetadataDuePurls, + insertDailyDownloads, + insertPackagistPackages, + logAuditFieldChanges, + markPackagist30dProcessed, + markPackagistDailyProcessed, + markPackagistMetadataScanned, +} from '@crowd/data-access-layer/src/packages' +import type { + PackagistDailyCandidate, + PackagistMetadataCandidate, + PackagistRunResult, +} from '@crowd/data-access-layer/src/packages/packagistPackageState' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { getServiceChildLogger } from '@crowd/logging' + +import { getPackagesDb } from '../db' +import { mapWithConcurrency } from '../utils/concurrency' +import { isClientError } from '../utils/isClientError' + +import { persistPackagist30dWindow } from './downloads' +import { expandComposerMetadata } from './expandMetadata' +import { fetchPackagistP2, fetchPackagistStats } from './fetchPackage' +import { fetchPackagistPackageList, parsePackagistPackageList } from './listPackages' +import { normalizePackagistStats, packagistNameFromPurl } from './normalize' +import { INGEST_MAX_ATTEMPTS } from './retryPolicy' +import { FetchError, isFetchError, isP2NotModified } from './types' +import { persistPackagistMetadata } from './upsertMetadata' +import { persistPackagistPackageInfo } from './upsertPackageInfo' + +const log = getServiceChildLogger('packagist') + +const WORKER = 'packagist' + +// 4xx/malformed get a few quick in-lane retries with a small linear backoff +const INGEST_4XX_ATTEMPTS = 3 +const INGEST_4XX_BACKOFF_MS = 1000 + +// Concurrency cap for the dynamic (packagist.org) endpoint, shared by all lanes +// that fetch it: metadata, downloads-30d, and daily downloads. +function statsConcurrency(): number { + const n = parseInt(process.env.CROWD_PACKAGES_PACKAGIST_STATS_CONCURRENCY ?? '10', 10) + return Math.max(1, Math.min(10, Number.isFinite(n) ? n : 10)) +} + +function metadataRefreshDays(): number { + const n = parseInt(process.env.CROWD_PACKAGES_PACKAGIST_METADATA_REFRESH_DAYS ?? '7', 10) + return Number.isFinite(n) && n > 0 ? n : 7 +} + +// Scope of the metadata sweep. Deliberately diverges from pypi (critical-only steady +// state): Packagist enriches ALL packages because deps.dev has no Packagist data to +// fall back on. Set CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL=true to narrow +// back to the critical slice. +function runOnlyForCritical(): boolean { + const raw = (process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL ?? 'false') + .trim() + .toLowerCase() + return raw === 'true' || raw === '1' || raw === 'yes' +} + +export async function packagistStopAfterFirstPage(): Promise { + const raw = (process.env.CROWD_PACKAGES_PACKAGIST_STOP_AFTER_FIRST_PAGE ?? 'false') + .trim() + .toLowerCase() + return raw === 'true' || raw === '1' || raw === 'yes' +} + +// Deterministic cutoff source for the watermark-draining download workflows. +export async function packagistCurrentTimestamp(): Promise { + return new Date().toISOString() +} + +// Fetch with the shared fast-retry contract: transient/429 results throw so Temporal +// retries the batch; 4xx/malformed get INGEST_4XX_ATTEMPTS quick in-lane retries and +// then surface as a give-up `error` the caller records on the state row. +async function fetchWithFastRetry( + fetchOnce: () => Promise, + what: string, +): Promise<{ value: T; attempts: number } | { error: FetchError; attempts: number }> { + for (let attempt = 1; ; attempt++) { + const result = await fetchOnce() + + if (!isFetchError(result)) { + return { value: result, attempts: attempt } + } + + if (!isClientError(result.statusCode, result.kind) && result.kind !== 'MALFORMED') { + throw new Error(`Failed to fetch ${what}: ${result.message}`) + } + + if (attempt >= INGEST_4XX_ATTEMPTS) { + return { error: result, attempts: attempt } + } + + await timeout(attempt * INGEST_4XX_BACKOFF_MS) + } +} + +function giveUpResult(error: FetchError): PackagistRunResult { + return { + status: 'error', + attempts: INGEST_4XX_ATTEMPTS, + httpStatus: error.statusCode, + errorKind: error.kind, + message: error.message, + } +} + +// The merged enrichment lane: one pass per package fetches BOTH registry endpoints — +// the dynamic one (package info, repo link, maintainers) and p2 (versions, dependencies). +export async function ingestOnePackagistMetadata( + qx: QueryExecutor, + candidate: PackagistMetadataCandidate, +): Promise { + const name = packagistNameFromPurl(candidate.purl) + + // Phase 1: dynamic endpoint + const info = await fetchWithFastRetry( + () => fetchPackagistStats(name), + `Packagist stats for ${name}`, + ) + if ('error' in info) { + log.warn( + { purl: candidate.purl, statusCode: info.error.statusCode, kind: info.error.kind }, + 'packagist package info 4xx/malformed after fast retries — marking scanned and skipping', + ) + await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(info.error)) + return + } + + const stats = normalizePackagistStats(info.value.package) + const persistedInfo = await persistPackagistPackageInfo(qx, candidate.purl, stats) + const changedFields = [...persistedInfo.changedFields] + + // Phase 2: p2 endpoint + const p2 = await fetchWithFastRetry( + () => fetchPackagistP2(name, candidate.metadataLastModified), + `Packagist metadata for ${name}`, + ) + if ('error' in p2) { + log.warn( + { purl: candidate.purl, statusCode: p2.error.statusCode, kind: p2.error.kind }, + 'packagist metadata 4xx/malformed after fast retries — marking scanned and skipping', + ) + // Phase-1 writes are already committed — their audit rows must not be dropped. + await logAuditFieldChanges(qx, WORKER, candidate.purl, changedFields) + await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(p2.error)) + return + } + + let lastModified: string | null = null + if (!isP2NotModified(p2.value)) { + const expanded = expandComposerMetadata(p2.value.minifiedVersions) + const persistResult = await persistPackagistMetadata(qx, candidate.purl, expanded) + if (persistResult.unresolvedDependencyTargets > 0) { + log.debug( + { purl: candidate.purl, unresolved: persistResult.unresolvedDependencyTargets }, + 'packagist dependency targets not found in packages — edges skipped', + ) + } + changedFields.push(...persistResult.changedFields) + lastModified = p2.value.lastModified + } + + await logAuditFieldChanges(qx, WORKER, candidate.purl, changedFields) + if (lastModified) { + await markPackagistMetadataScanned( + qx, + candidate.purl, + { status: 'success', attempts: p2.attempts }, + lastModified, + ) + } else { + await markPackagistMetadataScanned(qx, candidate.purl, { + status: 'success', + attempts: p2.attempts, + }) + } +} + +// The monthly downloads-30d lane: dynamic fetch, one window row per purl per month. +export async function ingestOnePackagist30dWindow( + qx: QueryExecutor, + purl: string, + runDate: string, +): Promise { + const name = packagistNameFromPurl(purl) + + const info = await fetchWithFastRetry( + () => fetchPackagistStats(name), + `Packagist stats for ${name}`, + ) + if ('error' in info) { + log.warn( + { purl, statusCode: info.error.statusCode, kind: info.error.kind }, + 'packagist 30d downloads 4xx/malformed after fast retries — marking processed and skipping', + ) + await markPackagist30dProcessed(qx, purl, giveUpResult(info.error)) + return + } + + await persistPackagist30dWindow(qx, purl, info.value.package.downloads?.monthly ?? null, runDate) + await markPackagist30dProcessed(qx, purl, { status: 'success', attempts: info.attempts }) +} + +// The daily downloads lane (critical slice): dynamic fetch, one downloads_daily row. +export async function ingestOnePackagistDailyDownload( + qx: QueryExecutor, + candidate: PackagistDailyCandidate, + runDate: string, +): Promise { + const name = packagistNameFromPurl(candidate.purl) + + const info = await fetchWithFastRetry( + () => fetchPackagistStats(name), + `Packagist stats for ${name}`, + ) + if ('error' in info) { + log.warn( + { purl: candidate.purl, statusCode: info.error.statusCode, kind: info.error.kind }, + 'packagist daily downloads 4xx/malformed after fast retries — marking processed and skipping', + ) + await markPackagistDailyProcessed(qx, candidate.purl, giveUpResult(info.error)) + return + } + + const daily = info.value.package.downloads?.daily + if (typeof daily === 'number') { + await insertDailyDownloads(qx, candidate.packageId, [{ day: runDate, downloads: daily }]) + } + await markPackagistDailyProcessed(qx, candidate.purl, { + status: 'success', + attempts: info.attempts, + }) +} + +export async function ingestPackagistItemsConcurrently( + items: T[], + attempt: number, + concurrency: number, + ingest: (item: T) => Promise, + onGiveUp: (item: T, err: unknown) => Promise, +): Promise { + await mapWithConcurrency(items, concurrency, async (item) => { + try { + await ingest(item) + } catch (err) { + // Retry via Temporal while attempts remain; then give up and continue + if (attempt < INGEST_MAX_ATTEMPTS) throw err + log.warn( + { item: String(item), attempt, err: String(err) }, + 'packagist item failed after max attempts — giving up', + ) + await onGiveUp(item, err) + } + }) +} + +export async function runPackagistPackageSeed(): Promise<{ discovered: number; invalid: number }> { + const result = await fetchPackagistPackageList() + + if (isFetchError(result)) { + throw new Error(`Failed to fetch Packagist package list: ${result.message}`) + } + + const { entries, invalid } = parsePackagistPackageList(result) + + if (entries.length > 0) { + const qx = await getPackagesDb() + for (const chunk of partition(entries, 5000)) { + await insertPackagistPackages(qx, chunk) + } + } + + return { discovered: entries.length, invalid } +} + +export async function getPackagistMetadataBatch( + afterPurl: string, + batchSize: number, +): Promise<{ candidates: PackagistMetadataCandidate[]; nextCursor: string }> { + const qx = await getPackagesDb() + const candidates = await getPackagistMetadataDuePurls( + qx, + afterPurl, + batchSize, + metadataRefreshDays(), + runOnlyForCritical(), + ) + return { + candidates, + nextCursor: candidates.length ? candidates[candidates.length - 1].purl : afterPurl, + } +} + +export async function ingestPackagistMetadataBatch( + candidates: PackagistMetadataCandidate[], +): Promise { + if (candidates.length === 0) return + const qx = await getPackagesDb() + const attempt = Context.current().info.attempt + + // The merged lane starts every ingest with a DYNAMIC-endpoint fetch, so it is + // bounded by that endpoint's 10-concurrent limit — not p2's 20. Running hotter + // gets connections reset by packagist.org ("fetch failed"). + await ingestPackagistItemsConcurrently( + candidates, + attempt, + statsConcurrency(), + (candidate) => ingestOnePackagistMetadata(qx, candidate), + (candidate, err) => + markPackagistMetadataScanned(qx, candidate.purl, { + status: 'error', + attempts: attempt, + message: String(err), + }), + ) + + log.info({ count: candidates.length }, 'Ingested Packagist metadata batch') +} + +export async function getPackagist30dBatch(cutoff: string, batchSize: number): Promise { + const qx = await getPackagesDb() + return getPackagist30dDuePurls(qx, cutoff, batchSize) +} + +export async function ingestPackagist30dBatch(purls: string[]): Promise { + if (purls.length === 0) return + const qx = await getPackagesDb() + const runDate = new Date().toISOString().slice(0, 10) + const attempt = Context.current().info.attempt + + await ingestPackagistItemsConcurrently( + purls, + attempt, + statsConcurrency(), + (purl) => ingestOnePackagist30dWindow(qx, purl, runDate), + (purl, err) => + markPackagist30dProcessed(qx, purl, { + status: 'error', + attempts: attempt, + message: String(err), + }), + ) + + log.info({ count: purls.length }, 'Ingested Packagist 30d downloads batch') +} + +export async function getPackagistDailyBatch( + cutoff: string, + batchSize: number, +): Promise { + const qx = await getPackagesDb() + return getPackagistDailyDownloadsDue(qx, cutoff, batchSize) +} + +export async function ingestPackagistDailyBatch( + candidates: PackagistDailyCandidate[], +): Promise { + if (candidates.length === 0) return + const qx = await getPackagesDb() + const runDate = new Date().toISOString().slice(0, 10) + const attempt = Context.current().info.attempt + + await ingestPackagistItemsConcurrently( + candidates, + attempt, + statsConcurrency(), + (candidate) => ingestOnePackagistDailyDownload(qx, candidate, runDate), + (candidate, err) => + markPackagistDailyProcessed(qx, candidate.purl, { + status: 'error', + attempts: attempt, + message: String(err), + }), + ) + + log.info({ count: candidates.length }, 'Ingested Packagist daily downloads batch') +} + +export async function getCriticalPackagistCount(): Promise { + const qx = await getPackagesDb() + return getCriticalPackagistPackageCount(qx) +} diff --git a/services/apps/packages_worker/src/packagist/downloads.ts b/services/apps/packages_worker/src/packagist/downloads.ts new file mode 100644 index 0000000000..e868c7b56e --- /dev/null +++ b/services/apps/packages_worker/src/packagist/downloads.ts @@ -0,0 +1,38 @@ +import { + getExistingLast30dEndDates, + upsertLast30dDownload, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +// Compute the monthly window for observed rolling 30d downloads. +// endDate = first of the run month; startDate = endDate minus 30 days. +export function monthlyWindowFor(runDate: string): { startDate: string; endDate: string } { + const d = new Date(runDate + 'T00:00:00Z') + // endDate: first of the current month + const endDate = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), 1)) + // startDate: endDate minus 30 days + const startDate = new Date(endDate.getTime() - 30 * 24 * 60 * 60 * 1000) + + return { + startDate: startDate.toISOString().slice(0, 10), + endDate: endDate.toISOString().slice(0, 10), + } +} + +// One window row per purl per month, mirrored to packages.downloads_last_30d +// (npm parity). A window already recorded for the month is never overwritten — +// the value is the observation closest to the boundary. +export async function persistPackagist30dWindow( + qx: QueryExecutor, + purl: string, + monthly: number | null, + runDate: string, +): Promise { + if (monthly == null) return + + const { startDate, endDate } = monthlyWindowFor(runDate) + const existing = await getExistingLast30dEndDates(qx, purl, endDate, endDate) + if (existing.length === 0) { + await upsertLast30dDownload(qx, purl, startDate, endDate, monthly, true) + } +} diff --git a/services/apps/packages_worker/src/packagist/expandMetadata.ts b/services/apps/packages_worker/src/packagist/expandMetadata.ts new file mode 100644 index 0000000000..2acb74278e --- /dev/null +++ b/services/apps/packages_worker/src/packagist/expandMetadata.ts @@ -0,0 +1,29 @@ +import type { PackagistExpandedVersion, PackagistMinifiedVersion } from './types' + +export function expandComposerMetadata( + versions: PackagistMinifiedVersion[], +): PackagistExpandedVersion[] { + if (versions.length === 0) return [] + + const result: PackagistExpandedVersion[] = [] + let current: Record = {} + + for (const minified of versions) { + // Merge: start with previous expanded, apply the minified diff + const next = { ...current } + for (const [key, value] of Object.entries(minified)) { + if (value === '__unset') { + delete next[key] + } else { + next[key] = value + } + } + + // `next` is never mutated after this — the following iteration spreads it + // into a fresh object — so it can be pushed directly. + result.push(next as PackagistExpandedVersion) + current = next + } + + return result +} diff --git a/services/apps/packages_worker/src/packagist/fetchPackage.ts b/services/apps/packages_worker/src/packagist/fetchPackage.ts new file mode 100644 index 0000000000..f791de9c90 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/fetchPackage.ts @@ -0,0 +1,141 @@ +import type { + FetchError, + P2Metadata, + P2NotModified, + PackagistMinifiedVersion, + PackagistStatsJson, +} from './types' + +export function buildPackagistUserAgent(): string { + const mailto = process.env.CROWD_PACKAGES_PACKAGIST_MAILTO || 'oss-packages@linuxfoundation.org' + return `lfx-packages-worker/0.1 (+https://lfx.linuxfoundation.org; mailto=${mailto})` +} + +// undici wraps network failures as `TypeError: fetch failed` with the real reason +// (ECONNRESET, ETIMEDOUT, DNS, …) on `cause` — surface it or the logs are undiagnosable. +export function describeFetchFailure(err: unknown): string { + const cause = (err as { cause?: { message?: string; code?: string } })?.cause + const detail = cause?.code ?? cause?.message + return detail ? `${String(err)} (cause: ${detail})` : String(err) +} + +export async function fetchPackagistStats(name: string): Promise { + const url = `https://packagist.org/packages/${name}.json` + const abort = new AbortController() + const timer = setTimeout(() => abort.abort(), 30_000) + + try { + let res: Response + try { + res = await fetch(url, { + headers: { + Accept: 'application/json', + 'User-Agent': buildPackagistUserAgent(), + }, + signal: abort.signal, + }) + } catch (err) { + return { kind: 'TRANSIENT', message: describeFetchFailure(err) } + } + + if (res.status === 404) + return { kind: 'NOT_FOUND', message: `${name} not found`, statusCode: 404 } + if (res.status === 429) return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429 } + if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + + let json: unknown + try { + json = await res.json() + } catch { + if (abort.signal.aborted) return { kind: 'TRANSIENT', message: 'body read timed out' } + return { kind: 'MALFORMED', message: 'invalid JSON' } + } + + // Shape guard: body.package must be an object with a string name + if (!isPackagistStatsJson(json)) { + return { kind: 'MALFORMED', message: 'unexpected shape' } + } + + return json + } finally { + clearTimeout(timer) + } +} + +export async function fetchPackagistP2( + name: string, + ifModifiedSince: string | null, +): Promise { + const url = `https://repo.packagist.org/p2/${name}.json` + const abort = new AbortController() + const timer = setTimeout(() => abort.abort(), 30_000) + + try { + const headers: Record = { + Accept: 'application/json', + 'User-Agent': buildPackagistUserAgent(), + } + if (ifModifiedSince !== null) { + headers['If-Modified-Since'] = ifModifiedSince + } + + let res: Response + try { + res = await fetch(url, { + headers, + signal: abort.signal, + }) + } catch (err) { + return { kind: 'TRANSIENT', message: describeFetchFailure(err) } + } + + // 304 must be checked first + if (res.status === 304) { + return { kind: 'NOT_MODIFIED' } + } + + if (res.status === 404) + return { kind: 'NOT_FOUND', message: `${name} not found`, statusCode: 404 } + if (res.status === 429) return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429 } + if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + + let json: unknown + try { + json = await res.json() + } catch { + if (abort.signal.aborted) return { kind: 'TRANSIENT', message: 'body read timed out' } + return { kind: 'MALFORMED', message: 'invalid JSON' } + } + + // Shape guard: body.packages[name] must be an array + if (!isP2Response(json, name)) { + return { kind: 'MALFORMED', message: 'unexpected shape' } + } + + const minifiedVersions = json.packages[name] as PackagistMinifiedVersion[] + const lastModified = res.headers.get('last-modified') ?? null + + return { + minifiedVersions, + lastModified, + } + } finally { + clearTimeout(timer) + } +} + +function isPackagistStatsJson(v: unknown): v is PackagistStatsJson { + if (typeof v !== 'object' || v === null || !('package' in v)) return false + const pkg = (v as { package: unknown }).package + return ( + typeof pkg === 'object' && pkg !== null && typeof (pkg as { name?: unknown }).name === 'string' + ) +} + +function isP2Response(v: unknown, name: string): v is { packages: Record } { + if (typeof v !== 'object' || v === null || !('packages' in v)) return false + const packages = (v as { packages: unknown }).packages + if (typeof packages !== 'object' || packages === null) return false + const entries = (packages as Record)[name] + return Array.isArray(entries) +} diff --git a/services/apps/packages_worker/src/packagist/listPackages.ts b/services/apps/packages_worker/src/packagist/listPackages.ts new file mode 100644 index 0000000000..4c398d756b --- /dev/null +++ b/services/apps/packages_worker/src/packagist/listPackages.ts @@ -0,0 +1,103 @@ +import { buildPackagistUserAgent, describeFetchFailure } from './fetchPackage' +import type { FetchError } from './types' + +const PACKAGIST_LIST = 'https://packagist.org/packages/list.json' +const COMPOSER_NAME_REGEX = /^[a-z0-9]([_.-]?[a-z0-9]+)*$/ + +export interface PackagistListEntry { + vendor: string + name: string + purl: string +} + +export function parsePackagistPackageList(json: unknown): { + entries: PackagistListEntry[] + invalid: number +} { + if (typeof json !== 'object' || json === null) { + throw new TypeError('list.json must be an object') + } + + const root = json as { packageNames?: unknown } + if (!Array.isArray(root.packageNames)) { + throw new TypeError('packageNames must be an array') + } + + const seen = new Set() + const entries: PackagistListEntry[] = [] + let invalid = 0 + + for (const item of root.packageNames) { + if (typeof item !== 'string') { + invalid++ + continue + } + + const lowercased = item.toLowerCase() + const parts = lowercased.split('/') + + // Validate: exactly one slash, each side non-empty and matches composer name pattern + if ( + parts.length !== 2 || + !parts[0] || + !parts[1] || + !COMPOSER_NAME_REGEX.test(parts[0]) || + !COMPOSER_NAME_REGEX.test(parts[1]) + ) { + invalid++ + continue + } + + // Dedup on the lowercased form + if (seen.has(lowercased)) { + continue + } + seen.add(lowercased) + + entries.push({ + vendor: parts[0], + name: parts[1], + purl: `pkg:composer/${parts[0]}/${parts[1]}`, + }) + } + + return { entries, invalid } +} + +export async function fetchPackagistPackageList(): Promise { + const abort = new AbortController() + // 30s timer covering the body read too + const timer = setTimeout(() => abort.abort(), 30_000) + + try { + let res: Response + try { + res = await fetch(PACKAGIST_LIST, { + headers: { + Accept: 'application/json', + 'User-Agent': buildPackagistUserAgent(), + }, + signal: abort.signal, + }) + } catch (err) { + return { kind: 'TRANSIENT', message: describeFetchFailure(err) } + } + + // Status classification: 404 NOT_FOUND, 429 RATE_LIMIT, other non-ok TRANSIENT + if (res.status === 404) return { kind: 'NOT_FOUND', message: 'list not found', statusCode: 404 } + if (res.status === 429) return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429 } + if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + + let json: unknown + try { + json = await res.json() + } catch { + if (abort.signal.aborted) return { kind: 'TRANSIENT', message: 'body read timed out' } + return { kind: 'MALFORMED', message: 'invalid JSON' } + } + + return json + } finally { + clearTimeout(timer) + } +} diff --git a/services/apps/packages_worker/src/packagist/normalize.ts b/services/apps/packages_worker/src/packagist/normalize.ts new file mode 100644 index 0000000000..c73583191d --- /dev/null +++ b/services/apps/packages_worker/src/packagist/normalize.ts @@ -0,0 +1,216 @@ +import type { + NormalizedPackagistStats, + PackagistDependency, + PackagistExpandedVersion, + PackagistPackageInfo, + PackagistVersionRow, +} from './types' + +const PURL_COMPOSER_PREFIX = 'pkg:composer/' + +export function packagistNameFromPurl(purl: string): string { + return purl.slice(PURL_COMPOSER_PREFIX.length) +} + +export function purlFromPackagistName(name: string): string { + return PURL_COMPOSER_PREFIX + name +} + +function blankToNull(s: string | null | undefined): string | null { + if (s == null) return null + const t = s.trim() + return t || null +} + +export function normalizePackagistStats(pkg: PackagistPackageInfo): NormalizedPackagistStats { + const description = blankToNull(pkg.description) + const repositoryUrl = blankToNull(pkg.repository) + + // Determine status: abandoned → deprecated, else active + let status: 'active' | 'deprecated' = 'active' + if (pkg.abandoned === true || (typeof pkg.abandoned === 'string' && pkg.abandoned)) { + status = 'deprecated' + } + + // Extract downloads, defaulting to null if not a number + const downloadsTotal = typeof pkg.downloads?.total === 'number' ? pkg.downloads.total : null + const downloadsMonthly = typeof pkg.downloads?.monthly === 'number' ? pkg.downloads.monthly : null + + const dependents = typeof pkg.dependents === 'number' ? pkg.dependents : null + + // Extract maintainers with non-empty names + const maintainers = (pkg.maintainers ?? []) + .filter((m): m is { name: string } => typeof m.name === 'string' && !!m.name) + .map((m) => ({ + username: m.name, + displayName: null, + email: null, + role: 'maintainer' as const, + })) + + return { + name: pkg.name, + description, + repositoryUrl, + status, + dependents, + downloadsTotal, + downloadsMonthly, + maintainers, + } +} + +export function isPackagistDevVersion(version: string, versionNormalized?: string): boolean { + if (version.startsWith('dev-')) return true + if (versionNormalized?.endsWith('-dev')) return true + return false +} + +export function isPackagistPrerelease(versionNormalized: string): boolean { + const parts = versionNormalized.split('-') + if (parts.length < 2) return false + + const suffix = parts[1].toLowerCase() + + // Match: alpha|a|beta|b|rc followed by optional digits + // Stable: plain number, -patch\d*, -p\d* + if (suffix.match(/^(alpha|a|beta|b|rc)\d*$/)) { + return true + } + + return false +} + +// Custom compare — the repo's semver/osv versionCompare can't be reused here. +// Composer's version_normalized is 4-part (e.g. 2.0.0.0); semver.parse returns null +// on it, and the osv wrapper deliberately refuses lossy coercion. +const PRERELEASE_RANKS: Record = { alpha: 1, a: 1, beta: 2, b: 2, rc: 3 } +const STABLE_RANK = 4 + +function parseComposerVersion(v: string): { base: number[]; rank: number; suffixNum: number } { + const dashIdx = v.indexOf('-') + const base = (dashIdx === -1 ? v : v.slice(0, dashIdx)).split('.').map((p) => { + const n = Number(p) + return Number.isFinite(n) ? n : 0 + }) + + const suffixMatch = + dashIdx === -1 ? null : v.slice(dashIdx + 1).match(/^(alpha|a|beta|b|rc)(\d*)/i) + if (!suffixMatch) return { base, rank: STABLE_RANK, suffixNum: 0 } + + return { + base, + rank: PRERELEASE_RANKS[suffixMatch[1].toLowerCase()], + suffixNum: suffixMatch[2] ? Number(suffixMatch[2]) : 0, + } +} + +function compareComposerVersions(a: string, b: string): number { + const pa = parseComposerVersion(a) + const pb = parseComposerVersion(b) + + const maxLen = Math.max(pa.base.length, pb.base.length) + for (let i = 0; i < maxLen; i++) { + const diff = (pa.base[i] ?? 0) - (pb.base[i] ?? 0) + if (diff !== 0) return diff + } + + if (pa.rank !== pb.rank) return pa.rank - pb.rank + return pa.suffixNum - pb.suffixNum +} + +export function buildPackagistVersionRows(versions: PackagistExpandedVersion[]): { + versionRows: PackagistVersionRow[] + latestVersion: string | null + firstReleaseAt: string | null + latestReleaseAt: string | null + licenses: string[] | null + homepage: string | null +} { + // Filter out dev versions + const kept = versions.filter((v) => !isPackagistDevVersion(v.version, v.version_normalized)) + + if (kept.length === 0) { + return { + versionRows: [], + latestVersion: null, + firstReleaseAt: null, + latestReleaseAt: null, + licenses: null, + homepage: null, + } + } + + const versionRows: PackagistVersionRow[] = kept.map((v) => ({ + number: v.version, + publishedAt: v.time ?? null, + isLatest: false, // Will be set below + isPrerelease: isPackagistPrerelease(v.version_normalized ?? v.version), + license: (v.license && v.license.length > 0 ? v.license[0] : null) ?? null, + })) + + // Find latest: prefer stable over prerelease; within each group use Composer ordering + let latestIdx = 0 + + for (let i = 1; i < kept.length; i++) { + if (versionRows[i].isPrerelease !== versionRows[latestIdx].isPrerelease) { + // A stable release always beats a prerelease + if (!versionRows[i].isPrerelease) latestIdx = i + continue + } + + const currentNorm = kept[i].version_normalized ?? kept[i].version + const latestNorm = kept[latestIdx].version_normalized ?? kept[latestIdx].version + if (compareComposerVersions(currentNorm, latestNorm) > 0) latestIdx = i + } + + versionRows[latestIdx].isLatest = true + + // Compute aggregates + const times = kept + .map((v) => v.time) + .filter((t): t is string => t != null) + .sort() + + const firstReleaseAt = times[0] ?? null + const latestReleaseAt = times[times.length - 1] ?? null + + // Licenses and homepage come from the latest row + const licenses = kept[latestIdx].license?.length ? (kept[latestIdx].license ?? null) : null + const homepage = blankToNull(kept[latestIdx].homepage) + + return { + versionRows, + latestVersion: kept[latestIdx].version, + firstReleaseAt, + latestReleaseAt, + licenses, + homepage, + } +} + +export function extractVersionDependencies( + version: PackagistExpandedVersion, +): PackagistDependency[] { + const deps: PackagistDependency[] = [] + + // Add require (direct) dependencies, excluding platform targets + if (version.require && typeof version.require === 'object') { + for (const [name, constraint] of Object.entries(version.require)) { + if (typeof constraint === 'string' && name.includes('/')) { + deps.push({ name, constraint, kind: 'direct' }) + } + } + } + + // Add require-dev (dev) dependencies, excluding platform targets + if (version['require-dev'] && typeof version['require-dev'] === 'object') { + for (const [name, constraint] of Object.entries(version['require-dev'])) { + if (typeof constraint === 'string' && name.includes('/')) { + deps.push({ name, constraint, kind: 'dev' }) + } + } + } + + return deps +} diff --git a/services/apps/packages_worker/src/packagist/retryPolicy.ts b/services/apps/packages_worker/src/packagist/retryPolicy.ts new file mode 100644 index 0000000000..08ecd8fd56 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/retryPolicy.ts @@ -0,0 +1,4 @@ +// Per-package fetch attempts. Shared between the workflow's Temporal retry policy and the +// activity's give-up threshold so the two never drift: a package is only given up on +// (marked scanned-error so the cursor can advance) once Temporal has exhausted these attempts. +export const INGEST_MAX_ATTEMPTS = 5 diff --git a/services/apps/packages_worker/src/packagist/schedule.ts b/services/apps/packages_worker/src/packagist/schedule.ts new file mode 100644 index 0000000000..9cc646792b --- /dev/null +++ b/services/apps/packages_worker/src/packagist/schedule.ts @@ -0,0 +1,72 @@ +import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/client' + +// Cron minutes deliberately off :00 per Packagist crawler guidelines. +// The weekly metadata drain has no cron: seedPackagistPackages starts it as a +// child workflow once the seed completes, so sequencing is an event, not a +// clock offset. +export const PACKAGIST_CRONS = { + seed: '17 2 * * 0', + downloads30d: '53 3 1 * *', + downloadsDaily: '23 6 * * *', +} + +// Workflow types by name (not function reference) so this module doesn't pull the +// whole workflows index into consumers that only need the crons. +const SCHEDULES = [ + { + scheduleId: 'packagist-seed', + cron: PACKAGIST_CRONS.seed, + workflowType: 'seedPackagistPackages', + args: [] as unknown[], + }, + { + scheduleId: 'packagist-downloads-30d', + cron: PACKAGIST_CRONS.downloads30d, + workflowType: 'ingestPackagistDownloads30d', + args: [{}] as unknown[], + }, + { + scheduleId: 'packagist-downloads-daily', + cron: PACKAGIST_CRONS.downloadsDaily, + workflowType: 'ingestPackagistDownloadsDaily', + args: [{}] as unknown[], + }, +] + +export async function schedulePackagistIngest(): Promise { + // svc is imported lazily: its module graph can't load under vitest, and the wiring + // test imports PACKAGIST_CRONS from this file. The .js extension is required by + // node16 module resolution for dynamic imports. + const { svc } = await import('../service.js') + + const { temporal } = svc + if (!temporal) throw new Error('Temporal client not initialized') + + for (const schedule of SCHEDULES) { + try { + await temporal.schedule.create({ + scheduleId: schedule.scheduleId, + spec: { + cronExpressions: [schedule.cron], + }, + policies: { + overlap: ScheduleOverlapPolicy.SKIP, + catchupWindow: '1 hour', + }, + action: { + type: 'startWorkflow', + workflowType: schedule.workflowType, + taskQueue: 'packagist-worker', + workflowRunTimeout: '24 hours', + args: schedule.args, + }, + }) + } catch (err) { + if (err instanceof ScheduleAlreadyRunning) { + svc.log.info(`Schedule ${schedule.scheduleId} already registered.`) + } else { + throw err + } + } + } +} diff --git a/services/apps/packages_worker/src/packagist/types.ts b/services/apps/packages_worker/src/packagist/types.ts new file mode 100644 index 0000000000..dbb7baa360 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/types.ts @@ -0,0 +1,115 @@ +import type { PackageMaintainerInput } from '@crowd/data-access-layer/src/packages/maintainers' + +export type FetchErrorKind = 'RATE_LIMIT' | 'TRANSIENT' | 'NOT_FOUND' | 'MALFORMED' + +export interface FetchError { + kind: FetchErrorKind + message: string + statusCode?: number +} + +const FETCH_ERROR_KINDS: ReadonlySet = new Set([ + 'RATE_LIMIT', + 'TRANSIENT', + 'NOT_FOUND', + 'MALFORMED', +]) + +// Explicit kind check (unlike pypi's loose `'kind' in v`) because the p2 fetch has a +// non-error NOT_MODIFIED outcome that also carries a `kind`. +export function isFetchError(v: unknown): v is FetchError { + return ( + typeof v === 'object' && + v !== null && + 'kind' in v && + FETCH_ERROR_KINDS.has((v as { kind: string }).kind) + ) +} + +// Subset of packagist.org/packages/{vendor}/{name}.json (dynamic stats endpoint) we consume. +export interface PackagistMaintainer { + name?: string + avatar_url?: string +} + +export interface PackagistDownloads { + total?: number + monthly?: number + daily?: number +} + +export interface PackagistPackageInfo { + name: string + description?: string | null + time?: string | null + maintainers?: PackagistMaintainer[] + type?: string | null + repository?: string | null + language?: string | null + // Absent/false when maintained; true or a replacement package name when abandoned. + abandoned?: boolean | string + dependents?: number + suggesters?: number + favers?: number + downloads?: PackagistDownloads +} + +export interface PackagistStatsJson { + package: PackagistPackageInfo +} + +export interface NormalizedPackagistStats { + // vendor/name, as Packagist reports it + name: string + description: string | null + repositoryUrl: string | null + status: 'active' | 'deprecated' + dependents: number | null + downloadsTotal: number | null + downloadsMonthly: number | null + maintainers: PackageMaintainerInput[] +} + +// repo.packagist.org/p2/{vendor}/{name}.json — Composer-minified version objects. +// The first object is complete; each subsequent one carries only changed keys, with +// the literal string '__unset' marking a removed key. +export type PackagistMinifiedVersion = { version: string } & Record + +export interface PackagistExpandedVersion { + version: string + version_normalized?: string + time?: string + license?: string[] + homepage?: string + require?: Record + 'require-dev'?: Record + [key: string]: unknown +} + +export interface P2Metadata { + minifiedVersions: PackagistMinifiedVersion[] + // Last-Modified response header, replayed as If-Modified-Since on the next fetch. + lastModified: string | null +} + +export interface P2NotModified { + kind: 'NOT_MODIFIED' +} + +export function isP2NotModified(v: unknown): v is P2NotModified { + return typeof v === 'object' && v !== null && (v as { kind?: string }).kind === 'NOT_MODIFIED' +} + +export interface PackagistVersionRow { + number: string + publishedAt: string | null + isLatest: boolean + isPrerelease: boolean + license: string | null +} + +export interface PackagistDependency { + name: string + constraint: string + kind: 'direct' | 'dev' +} diff --git a/services/apps/packages_worker/src/packagist/upsertMetadata.ts b/services/apps/packages_worker/src/packagist/upsertMetadata.ts new file mode 100644 index 0000000000..2f7520ab7c --- /dev/null +++ b/services/apps/packages_worker/src/packagist/upsertMetadata.ts @@ -0,0 +1,90 @@ +import { + getPackagistPackageIdsByNames, + updatePackagistVersionAggregates, + upsertPackagistVersions, + upsertVersionDependencies, +} from '@crowd/data-access-layer/src/packages' +import type { VersionDependencyEdge } from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { + buildPackagistVersionRows, + extractVersionDependencies, + isPackagistDevVersion, +} from './normalize' +import type { PackagistDependency, PackagistExpandedVersion } from './types' + +export async function persistPackagistMetadata( + qx: QueryExecutor, + purl: string, + expanded: PackagistExpandedVersion[], +): Promise<{ found: boolean; changedFields: string[]; unresolvedDependencyTargets: number }> { + const { versionRows, latestVersion, firstReleaseAt, latestReleaseAt, licenses, homepage } = + buildPackagistVersionRows(expanded) + + const agg = await updatePackagistVersionAggregates(qx, purl, { + versionsCount: versionRows.length, + latestVersion, + firstReleaseAt, + latestReleaseAt, + licenses, + homepage, + }) + + if (!agg) { + return { found: false, changedFields: [], unresolvedDependencyTargets: 0 } + } + + const changedFields = [...agg.changedFields] + + let versionIds: Array<{ number: string; id: string }> = [] + if (versionRows.length > 0) { + const versionResult = await upsertPackagistVersions(qx, agg.id, versionRows, latestVersion) + changedFields.push(...versionResult.changedFields) + versionIds = versionResult.versionIds + } + + // One pass over the tagged versions collects target names and provisional edges; + // target ids are then resolved in a single batch query. + const versionMap = new Map(versionIds.map((v) => [v.number, v.id])) + const targetNames = new Set() + const pending: Array<{ versionId: string; dep: PackagistDependency }> = [] + + for (const v of expanded) { + if (isPackagistDevVersion(v.version, v.version_normalized)) continue + const versionId = versionMap.get(v.version) + if (!versionId) continue + + for (const dep of extractVersionDependencies(v)) { + targetNames.add(dep.name) + pending.push({ versionId, dep }) + } + } + + let unresolvedDependencyTargets = 0 + if (pending.length > 0) { + const idMap = await getPackagistPackageIdsByNames(qx, Array.from(targetNames)) + + const edges: VersionDependencyEdge[] = [] + for (const { versionId, dep } of pending) { + const dependsOnId = idMap.get(dep.name) + if (!dependsOnId) { + unresolvedDependencyTargets++ + continue + } + edges.push({ + packageId: agg.id, + versionId, + dependsOnId, + constraint: dep.constraint, + kind: dep.kind, + }) + } + + if (edges.length > 0) { + await upsertVersionDependencies(qx, edges) + } + } + + return { found: true, changedFields, unresolvedDependencyTargets } +} diff --git a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts new file mode 100644 index 0000000000..cc179f9238 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts @@ -0,0 +1,55 @@ +import { + getOrCreateRepoByUrl, + updatePackagistPackageStats, + upsertPackageMaintainers, + upsertPackageRepo, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl' + +import type { NormalizedPackagistStats } from './types' + +// Dynamic-endpoint persistence: packages fields + repo link for ALL packages, +// maintainers only for critical ones. Download rows are NOT written here — they +// belong to the dedicated downloads-30d/daily lanes. +export async function persistPackagistPackageInfo( + qx: QueryExecutor, + purl: string, + stats: NormalizedPackagistStats, +): Promise<{ found: boolean; changedFields: string[] }> { + // Step 1: Update packages row + const canonical = stats.repositoryUrl ? canonicalizeRepoUrl(stats.repositoryUrl) : null + const result = await updatePackagistPackageStats(qx, { + purl, + description: stats.description, + declaredRepositoryUrl: stats.repositoryUrl, + repositoryUrl: canonical?.url ?? null, + status: stats.status, + downloadsLast30d: stats.downloadsMonthly, + totalDownloads: stats.downloadsTotal, + dependentCount: stats.dependents, + }) + + if (!result) { + return { found: false, changedFields: [] } + } + + const { id, isCritical } = result + const changedFields = [...result.changedFields] + + // Step 2: Link the repo for ALL packages — 'declared'/0.8 is the manifest-declared + // convention shared by npm/pypi/maven/cargo + if (canonical) { + const repo = await getOrCreateRepoByUrl(qx, canonical.url, canonical.host) + const linkChanged = await upsertPackageRepo(qx, id, repo.id, 'declared', 0.8) + changedFields.push(...repo.changedFields, ...linkChanged) + } + + // Step 3: Maintainers only for critical packages + if (isCritical && stats.maintainers.length > 0) { + await upsertPackageMaintainers(qx, id, stats.maintainers, 'packagist') + } + + return { found: true, changedFields } +} diff --git a/services/apps/packages_worker/src/packagist/workflows.ts b/services/apps/packages_worker/src/packagist/workflows.ts new file mode 100644 index 0000000000..9225126ef6 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/workflows.ts @@ -0,0 +1,89 @@ +import { ParentClosePolicy, continueAsNew, proxyActivities, startChild } from '@temporalio/workflow' + +import type * as activities from './activities' +import { INGEST_MAX_ATTEMPTS } from './retryPolicy' + +const acts = proxyActivities({ + startToCloseTimeout: '15 minutes', + retry: { + initialInterval: '30 seconds', + backoffCoefficient: 2, + maximumAttempts: INGEST_MAX_ATTEMPTS, + }, +}) + +const INGEST_BATCH = 50 +const ROUNDS_PER_RUN = 20 + +interface MetadataState { + cursor?: string +} + +interface DownloadsState { + cutoff?: string +} + +export async function seedPackagistPackages(): Promise { + await acts.runPackagistPackageSeed() + + // Chain the metadata drain off a completed seed instead of a second cron: newly + // discovered packages are rows before the p2 crawl resolves dependency targets. + // Started (not awaited) with ABANDON — the drain runs for hours and must outlive + // this workflow; the state watermarks make the pass self-advancing regardless. + await startChild(ingestPackagistMetadata, { + args: [{}], + parentClosePolicy: ParentClosePolicy.ABANDON, + }) +} + +export async function ingestPackagistMetadata(state: MetadataState = {}): Promise { + let cursor = state.cursor || '' + const stopAfterFirstPage = await acts.packagistStopAfterFirstPage() + + for (let r = 0; r < ROUNDS_PER_RUN; r++) { + const { candidates, nextCursor } = await acts.getPackagistMetadataBatch(cursor, INGEST_BATCH) + if (candidates.length === 0) return + await acts.ingestPackagistMetadataBatch(candidates) + cursor = nextCursor + if (stopAfterFirstPage) return + if (candidates.length < INGEST_BATCH) return + } + + await continueAsNew({ cursor }) +} + +// Monthly capture of the observed rolling 30d window for every packagist package. +// The cutoff is fixed once per run (deterministic activity) so the watermark-based +// due-selection drains the whole universe exactly once per cron fire. +export async function ingestPackagistDownloads30d(state: DownloadsState = {}): Promise { + const cutoff = state.cutoff ?? (await acts.packagistCurrentTimestamp()) + const stopAfterFirstPage = await acts.packagistStopAfterFirstPage() + + for (let r = 0; r < ROUNDS_PER_RUN; r++) { + const purls = await acts.getPackagist30dBatch(cutoff, INGEST_BATCH) + if (purls.length === 0) return + await acts.ingestPackagist30dBatch(purls) + if (stopAfterFirstPage) return + if (purls.length < INGEST_BATCH) return + } + + await continueAsNew({ cutoff }) +} + +// Daily downloads capture for the critical slice. +export async function ingestPackagistDownloadsDaily(state: DownloadsState = {}): Promise { + if ((await acts.getCriticalPackagistCount()) === 0) return + + const cutoff = state.cutoff ?? (await acts.packagistCurrentTimestamp()) + const stopAfterFirstPage = await acts.packagistStopAfterFirstPage() + + for (let r = 0; r < ROUNDS_PER_RUN; r++) { + const candidates = await acts.getPackagistDailyBatch(cutoff, INGEST_BATCH) + if (candidates.length === 0) return + await acts.ingestPackagistDailyBatch(candidates) + if (stopAfterFirstPage) return + if (candidates.length < INGEST_BATCH) return + } + + await continueAsNew({ cutoff }) +} diff --git a/services/apps/packages_worker/src/scripts/triggerPackagistSeed.ts b/services/apps/packages_worker/src/scripts/triggerPackagistSeed.ts new file mode 100644 index 0000000000..48da3b31e8 --- /dev/null +++ b/services/apps/packages_worker/src/scripts/triggerPackagistSeed.ts @@ -0,0 +1,82 @@ +import { TEMPORAL_CONFIG, getTemporalClient } from '@crowd/temporal' + +import { + ingestPackagistDownloads30d, + ingestPackagistDownloadsDaily, + ingestPackagistMetadata, + seedPackagistPackages, +} from '../packagist/workflows' + +const HELP = ` +Usage: trigger-packagist [seed|metadata|downloads-30d|downloads-daily] + +Arguments: + seed Fetch packagist.org/packages/list.json and seed the packages table (default) + metadata Crawl the dynamic (package info) + p2 (versions/dependencies) endpoints + downloads-30d Capture the observed rolling 30d window for every package + downloads-daily Capture daily downloads for the critical slice + +Examples: + pnpm trigger-packagist:local + pnpm trigger-packagist:local seed + pnpm trigger-packagist:local metadata + pnpm trigger-packagist:local downloads-30d + pnpm trigger-packagist:local downloads-daily +` + +const TARGETS = ['seed', 'metadata', 'downloads-30d', 'downloads-daily'] as const +type Target = (typeof TARGETS)[number] + +async function main(): Promise { + const args = process.argv.slice(2) + if (args.includes('--help') || args.includes('-h')) { + console.log(HELP) + process.exit(0) + } + + const target = (args[0] ?? 'seed') as Target + if (!TARGETS.includes(target)) { + console.error(`Unknown target "${target}". Use one of: ${TARGETS.join(', ')}.`) + process.exit(1) + } + + const cfg = TEMPORAL_CONFIG() + if (!cfg.serverUrl || !cfg.namespace) { + console.error('Missing CROWD_TEMPORAL_SERVER_URL or CROWD_TEMPORAL_NAMESPACE') + process.exit(1) + } + + const client = await getTemporalClient(cfg) + const now = Date.now() + + if (target === 'seed') { + const handle = await client.workflow.start(seedPackagistPackages, { + taskQueue: 'packagist-worker', + workflowId: `packagist-seed-manual-${now}`, + args: [], + }) + console.log(`Started workflow ${handle.workflowId}`) + return + } + + const workflow = + target === 'metadata' + ? ingestPackagistMetadata + : target === 'downloads-30d' + ? ingestPackagistDownloads30d + : ingestPackagistDownloadsDaily + + const handle = await client.workflow.start(workflow, { + taskQueue: 'packagist-worker', + workflowId: `packagist-${target}-manual-${now}`, + args: [{}], + }) + console.log(`Started workflow ${handle.workflowId}`) +} + +main() + .then(() => process.exit(0)) + .catch((err) => { + console.error('Failed to trigger packagist workflow:', err) + process.exit(1) + }) diff --git a/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts b/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts index 42b83cf330..98c7ff2654 100644 --- a/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts +++ b/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts @@ -16,14 +16,14 @@ type EcosystemFetcher = ( ) => Promise // Keyed by the lowercased packages.ecosystem value. go has no package-manifest contacts. -const FETCHERS: Record = { +export const FETCHERS: Record = { npm: fetchNpm, pypi: fetchPypi, maven: fetchMaven, cargo: fetchCargo, nuget: fetchNuget, rubygems: fetchRubygems, - composer: fetchComposer, + packagist: fetchComposer, } export const extractManifest: Extractor = async (target, deps) => { diff --git a/services/apps/packages_worker/src/workflows/index.ts b/services/apps/packages_worker/src/workflows/index.ts index 91cf431947..c94b09de49 100644 --- a/services/apps/packages_worker/src/workflows/index.ts +++ b/services/apps/packages_worker/src/workflows/index.ts @@ -26,3 +26,9 @@ export { } from '../pypi/downloads/ingestPypiDownloads' export { ingestNuGetPackages } from '../nuget/workflows' export { ingestSecurityContacts } from '../security-contacts/workflows' +export { + seedPackagistPackages, + ingestPackagistMetadata, + ingestPackagistDownloads30d, + ingestPackagistDownloadsDaily, +} from '../packagist/workflows' diff --git a/services/apps/packages_worker/vitest.config.ts b/services/apps/packages_worker/vitest.config.ts index 530cc66032..d3c0b9dc1e 100644 --- a/services/apps/packages_worker/vitest.config.ts +++ b/services/apps/packages_worker/vitest.config.ts @@ -1,13 +1,43 @@ import { defineConfig } from 'vitest/config' +const shared = { + environment: 'node' as const, + server: { + deps: { + inline: [/@crowd\//], + }, + }, +} + export default defineConfig({ test: { - environment: 'node', - include: ['src/**/*.test.ts'], - server: { - deps: { - inline: [/@crowd\//], + projects: [ + { + test: { + ...shared, + name: 'packagist', + include: ['src/packagist/**/*.test.ts'], + // getPackagesDb() builds a lazy pg-promise pool from these at activity call + // time; the packagist suites mock all queries so nothing ever connects. + // Scoped to this project so DB-gated integration suites elsewhere (which + // auto-skip when the real vars are absent) are unaffected. + env: { + CROWD_PACKAGES_DB_WRITE_HOST: '127.0.0.1', + CROWD_PACKAGES_DB_PORT: '5432', + CROWD_PACKAGES_DB_DATABASE: 'packages-test', + CROWD_PACKAGES_DB_USERNAME: 'test', + CROWD_PACKAGES_DB_PASSWORD: 'test', + }, + }, }, - }, + { + test: { + ...shared, + name: 'default', + include: ['src/**/*.test.ts'], + exclude: ['src/packagist/**'], + }, + }, + ], }, }) diff --git a/services/libs/data-access-layer/src/packages/dependencies.ts b/services/libs/data-access-layer/src/packages/dependencies.ts new file mode 100644 index 0000000000..38484bc326 --- /dev/null +++ b/services/libs/data-access-layer/src/packages/dependencies.ts @@ -0,0 +1,35 @@ +import { QueryExecutor } from '../queryExecutor' + +export interface VersionDependencyEdge { + packageId: string + versionId: string + dependsOnId: string + constraint: string + kind: 'direct' | 'dev' +} + +export async function upsertVersionDependencies( + qx: QueryExecutor, + edges: VersionDependencyEdge[], +): Promise { + if (edges.length === 0) return 0 + + const packageIds = edges.map((e) => e.packageId) + const versionIds = edges.map((e) => e.versionId) + const dependsOnIds = edges.map((e) => e.dependsOnId) + const constraints = edges.map((e) => e.constraint) + const kinds = edges.map((e) => e.kind) + + const result = await qx.result( + `INSERT INTO package_dependencies (package_id, version_id, depends_on_id, version_constraint, dependency_kind, is_optional, created_at, updated_at) + SELECT e.package_id, e.version_id, e.depends_on_id, e.version_constraint, e.dependency_kind, FALSE, NOW(), NOW() + FROM unnest($(packageIds)::bigint[], $(versionIds)::bigint[], $(dependsOnIds)::bigint[], + $(constraints)::text[], $(kinds)::text[]) + AS e(package_id, version_id, depends_on_id, version_constraint, dependency_kind) + ON CONFLICT (version_id, depends_on_id, dependency_kind) DO UPDATE SET + version_constraint = EXCLUDED.version_constraint, + updated_at = NOW()`, + { packageIds, versionIds, dependsOnIds, constraints, kinds }, + ) + return result +} diff --git a/services/libs/data-access-layer/src/packages/index.ts b/services/libs/data-access-layer/src/packages/index.ts index 13631858c0..00fca97d7b 100644 --- a/services/libs/data-access-layer/src/packages/index.ts +++ b/services/libs/data-access-layer/src/packages/index.ts @@ -1,10 +1,12 @@ export * from './auditFieldChanges' +export * from './dependencies' export * from './downloadsDaily' export * from './downloadsLast30d' export * from './fundingLinks' export * from './maintainers' export * from './npmPackageState' export * from './npmWorkerState' +export * from './packagistPackageState' export * from './pypiPackageState' export * from './packages' export * from './repos' diff --git a/services/libs/data-access-layer/src/packages/packages.ts b/services/libs/data-access-layer/src/packages/packages.ts index 3a18d0cd82..164305a41a 100644 --- a/services/libs/data-access-layer/src/packages/packages.ts +++ b/services/libs/data-access-layer/src/packages/packages.ts @@ -226,6 +226,175 @@ export async function getTrackedNpmPackages( })) } +export interface PackagistSeedRow { + purl: string + vendor: string + name: string +} + +export async function insertPackagistPackages( + qx: QueryExecutor, + rows: PackagistSeedRow[], +): Promise { + if (rows.length === 0) return 0 + const purls = rows.map((r) => r.purl) + const vendors = rows.map((r) => r.vendor) + const names = rows.map((r) => r.name) + const result = await qx.result( + `INSERT INTO packages (purl, ecosystem, namespace, name, registry_url, status, ingestion_source, last_synced_at, created_at) + SELECT r.purl, 'packagist', r.vendor, r.name, + 'https://packagist.org/packages/' || r.vendor || '/' || r.name, + 'active', 'packagist-registry', NOW(), NOW() + FROM unnest($(purls)::text[], $(vendors)::text[], $(names)::text[]) + AS r(purl, vendor, name) + ON CONFLICT (purl) DO NOTHING`, + { purls, vendors, names }, + ) + return result +} + +export interface PackagistStatsUpdateInput { + purl: string + description: string | null + declaredRepositoryUrl: string | null + repositoryUrl: string | null + status: string + downloadsLast30d: number | null + totalDownloads: number | null + dependentCount: number | null +} + +export async function updatePackagistPackageStats( + qx: QueryExecutor, + input: PackagistStatsUpdateInput, +): Promise<{ id: string; isCritical: boolean; changedFields: string[] } | null> { + const row: { id: string; is_critical: boolean; changed_fields: string[] } | undefined = + await qx.selectOneOrNone( + `WITH old AS ( + SELECT description, declared_repository_url, repository_url, status, + downloads_last_30d, total_downloads, dependent_count, ingestion_source + FROM packages WHERE purl = $(purl) AND ecosystem = 'packagist' + ), + ins AS ( + UPDATE packages SET + description = $(description), + declared_repository_url = $(declaredRepositoryUrl), + repository_url = $(repositoryUrl), + status = $(status), + downloads_last_30d = COALESCE($(downloadsLast30d), downloads_last_30d), + total_downloads = COALESCE($(totalDownloads), total_downloads), + dependent_count = COALESCE($(dependentCount), dependent_count), + last_synced_at = NOW() + WHERE purl = $(purl) AND ecosystem = 'packagist' + RETURNING id, is_critical, description, declared_repository_url, repository_url, status, + downloads_last_30d, total_downloads, dependent_count, ingestion_source + ) + SELECT ins.id::text AS id, ins.is_critical, + array_remove(ARRAY[ + CASE WHEN o.description IS DISTINCT FROM ins.description THEN 'packages.description' END, + CASE WHEN o.declared_repository_url IS DISTINCT FROM ins.declared_repository_url THEN 'packages.declared_repository_url' END, + CASE WHEN o.repository_url IS DISTINCT FROM ins.repository_url THEN 'packages.repository_url' END, + CASE WHEN o.status IS DISTINCT FROM ins.status THEN 'packages.status' END, + CASE WHEN o.downloads_last_30d IS DISTINCT FROM ins.downloads_last_30d THEN 'packages.downloads_last_30d' END, + CASE WHEN o.total_downloads IS DISTINCT FROM ins.total_downloads THEN 'packages.total_downloads' END, + CASE WHEN o.dependent_count IS DISTINCT FROM ins.dependent_count THEN 'packages.dependent_count' END, + CASE WHEN o.ingestion_source IS DISTINCT FROM ins.ingestion_source THEN 'packages.ingestion_source' END + ], NULL) AS changed_fields + FROM ins LEFT JOIN old o ON true`, + input, + ) + + if (!row) return null + return { id: row.id, isCritical: row.is_critical, changedFields: row.changed_fields } +} + +export interface PackagistVersionAggregates { + versionsCount: number + latestVersion: string | null + firstReleaseAt: string | null + latestReleaseAt: string | null + licenses: string[] | null + homepage: string | null +} + +export async function updatePackagistVersionAggregates( + qx: QueryExecutor, + purl: string, + agg: PackagistVersionAggregates, +): Promise<{ id: string; changedFields: string[] } | null> { + const row: { id: string; changed_fields: string[] } | undefined = await qx.selectOneOrNone( + `WITH old AS ( + SELECT versions_count, latest_version, first_release_at, latest_release_at, licenses, homepage, ingestion_source + FROM packages WHERE purl = $(purl) AND ecosystem = 'packagist' + ), + ins AS ( + UPDATE packages SET + versions_count = CASE WHEN $(versionsCount) = 0 THEN packages.versions_count ELSE $(versionsCount) END, + -- keep-on-zero like versions_count: a dev-only manifest must not null the + -- known latest while the count is preserved + latest_version = CASE WHEN $(versionsCount) = 0 THEN packages.latest_version ELSE $(latestVersion) END, + first_release_at = COALESCE($(firstReleaseAt), first_release_at), + latest_release_at = COALESCE($(latestReleaseAt), latest_release_at), + licenses = COALESCE($(licenses), licenses), + homepage = COALESCE($(homepage), homepage), + last_synced_at = NOW() + WHERE purl = $(purl) AND ecosystem = 'packagist' + RETURNING id, versions_count, latest_version, first_release_at, latest_release_at, licenses, homepage, ingestion_source + ) + SELECT ins.id::text AS id, + array_remove(ARRAY[ + CASE WHEN o.versions_count IS DISTINCT FROM ins.versions_count THEN 'packages.versions_count' END, + CASE WHEN o.latest_version IS DISTINCT FROM ins.latest_version THEN 'packages.latest_version' END, + CASE WHEN o.first_release_at IS DISTINCT FROM ins.first_release_at THEN 'packages.first_release_at' END, + CASE WHEN o.latest_release_at IS DISTINCT FROM ins.latest_release_at THEN 'packages.latest_release_at' END, + CASE WHEN o.licenses IS DISTINCT FROM ins.licenses THEN 'packages.licenses' END, + CASE WHEN o.homepage IS DISTINCT FROM ins.homepage THEN 'packages.homepage' END, + CASE WHEN o.ingestion_source IS DISTINCT FROM ins.ingestion_source THEN 'packages.ingestion_source' END + ], NULL) AS changed_fields + FROM ins LEFT JOIN old o ON true`, + { purl, ...agg }, + ) + + if (!row) return null + return { id: row.id, changedFields: row.changed_fields } +} + +export async function getPackagistPackageIdsByNames( + qx: QueryExecutor, + names: string[], +): Promise> { + // Composer names are case-insensitive but seed rows are stored lowercased — + // lowercase the lookup so a manifest declaring 'Monolog/Monolog' still resolves. + const lowerByOriginal = new Map(names.map((name) => [name, name.toLowerCase()])) + const purls = [...lowerByOriginal.values()].map((name) => `pkg:composer/${name}`) + + const rows: Array<{ purl: string; id: string }> = await qx.select( + `SELECT purl, id::text AS id FROM packages + WHERE ecosystem = 'packagist' AND purl = ANY($(purls)::text[])`, + { purls }, + ) + + const idByLower = new Map() + for (const row of rows) { + idByLower.set(row.purl.slice('pkg:composer/'.length), row.id) + } + + // Map back keyed by the ORIGINAL (caller-supplied) names + const map = new Map() + for (const [original, lower] of lowerByOriginal) { + const id = idByLower.get(lower) + if (id) map.set(original, id) + } + return map +} + +export async function getCriticalPackagistPackageCount(qx: QueryExecutor): Promise { + const row: { count: string } = await qx.selectOne( + `SELECT COUNT(*)::text AS count FROM packages WHERE ecosystem = 'packagist' AND is_critical = TRUE`, + ) + return Number(row.count) +} + // How many critical PyPI packages exist — a cheap guard so the daily downloads workflow can // skip its BigQuery scan entirely when there are none to ingest (the merge scopes to is_critical). export async function getCriticalPypiPackageCount(qx: QueryExecutor): Promise { diff --git a/services/libs/data-access-layer/src/packages/packagistPackageState.ts b/services/libs/data-access-layer/src/packages/packagistPackageState.ts new file mode 100644 index 0000000000..2765159e02 --- /dev/null +++ b/services/libs/data-access-layer/src/packages/packagistPackageState.ts @@ -0,0 +1,140 @@ +import { QueryExecutor } from '../queryExecutor' + +// Structured outcome of a packagist ingest run, stored as JSONB in +// packagist_package_state.{metadata,downloads_30d,daily_downloads}_run_result. +export interface PackagistRunResult { + status: 'success' | 'error' + attempts: number + httpStatus?: number + errorKind?: string + message?: string +} + +export interface PackagistMetadataCandidate { + purl: string + // Last-Modified from the previous p2 fetch, replayed as If-Modified-Since. + metadataLastModified: string | null +} + +export interface PackagistDailyCandidate { + purl: string + packageId: string +} + +export async function markPackagistMetadataScanned( + qx: QueryExecutor, + purl: string, + result: PackagistRunResult, + metadataLastModified?: string | null, +): Promise { + await qx.result( + `INSERT INTO packagist_package_state (purl, metadata_run_result, metadata_last_run_at, metadata_last_modified) + VALUES ($(purl), $(result)::jsonb, NOW(), $(metadataLastModified)) + ON CONFLICT (purl) DO UPDATE SET + metadata_run_result = EXCLUDED.metadata_run_result, + metadata_last_run_at = EXCLUDED.metadata_last_run_at, + metadata_last_modified = COALESCE(EXCLUDED.metadata_last_modified, packagist_package_state.metadata_last_modified)`, + { purl, result: JSON.stringify(result), metadataLastModified: metadataLastModified ?? null }, + ) +} + +export async function getPackagistMetadataDuePurls( + qx: QueryExecutor, + afterPurl: string, + batchSize: number, + refreshDays: number, + onlyCritical: boolean, +): Promise { + const rows: Array<{ purl: string; metadata_last_modified: string | null }> = await qx.select( + `SELECT p.purl, s.metadata_last_modified + FROM packages p + LEFT JOIN packagist_package_state s ON s.purl = p.purl + WHERE p.ecosystem = 'packagist' + AND (NOT $(onlyCritical) OR p.is_critical = TRUE) + AND p.purl > $(afterPurl) + AND ( + s.metadata_last_run_at IS NULL + OR s.metadata_last_run_at < NOW() - ($(refreshDays) || ' days')::interval + ) + ORDER BY p.purl + LIMIT $(batchSize)`, + { afterPurl, batchSize, refreshDays, onlyCritical }, + ) + return rows.map((r) => ({ + purl: r.purl, + metadataLastModified: r.metadata_last_modified, + })) +} + +export async function getPackagist30dDuePurls( + qx: QueryExecutor, + cutoff: string, + batchSize: number, +): Promise { + const rows: Array<{ purl: string }> = await qx.select( + `SELECT p.purl + FROM packages p + LEFT JOIN packagist_package_state s ON s.purl = p.purl + WHERE p.ecosystem = 'packagist' + AND ( + s.downloads_30d_last_run_at IS NULL + OR s.downloads_30d_last_run_at < $(cutoff)::timestamptz + ) + ORDER BY s.downloads_30d_last_run_at ASC NULLS FIRST, p.purl + LIMIT $(batchSize)`, + { cutoff, batchSize }, + ) + return rows.map((r) => r.purl) +} + +export async function markPackagist30dProcessed( + qx: QueryExecutor, + purl: string, + result: PackagistRunResult, +): Promise { + await qx.result( + `INSERT INTO packagist_package_state (purl, downloads_30d_run_result, downloads_30d_last_run_at) + VALUES ($(purl), $(result)::jsonb, NOW()) + ON CONFLICT (purl) DO UPDATE SET + downloads_30d_run_result = EXCLUDED.downloads_30d_run_result, + downloads_30d_last_run_at = EXCLUDED.downloads_30d_last_run_at`, + { purl, result: JSON.stringify(result) }, + ) +} + +export async function getPackagistDailyDownloadsDue( + qx: QueryExecutor, + cutoff: string, + batchSize: number, +): Promise { + const rows: Array<{ purl: string; package_id: string }> = await qx.select( + `SELECT p.purl, p.id::text AS package_id + FROM packages p + LEFT JOIN packagist_package_state s ON s.purl = p.purl + WHERE p.ecosystem = 'packagist' + AND p.is_critical = TRUE + AND ( + s.daily_downloads_last_run_at IS NULL + OR s.daily_downloads_last_run_at < $(cutoff)::timestamptz + ) + ORDER BY s.daily_downloads_last_run_at ASC NULLS FIRST, p.purl + LIMIT $(batchSize)`, + { cutoff, batchSize }, + ) + return rows.map((r) => ({ purl: r.purl, packageId: r.package_id })) +} + +export async function markPackagistDailyProcessed( + qx: QueryExecutor, + purl: string, + result: PackagistRunResult, +): Promise { + await qx.result( + `INSERT INTO packagist_package_state (purl, daily_downloads_run_result, daily_downloads_last_run_at) + VALUES ($(purl), $(result)::jsonb, NOW()) + ON CONFLICT (purl) DO UPDATE SET + daily_downloads_run_result = EXCLUDED.daily_downloads_run_result, + daily_downloads_last_run_at = EXCLUDED.daily_downloads_last_run_at`, + { purl, result: JSON.stringify(result) }, + ) +} diff --git a/services/libs/data-access-layer/src/packages/versions.ts b/services/libs/data-access-layer/src/packages/versions.ts index f3fff47bc8..c2d08e0784 100644 --- a/services/libs/data-access-layer/src/packages/versions.ts +++ b/services/libs/data-access-layer/src/packages/versions.ts @@ -69,6 +69,96 @@ export async function upsertNpmVersions( return row.changed_fields } +export interface PackagistVersionInput { + number: string + publishedAt: string | null + isLatest: boolean + isPrerelease: boolean + license: string | null +} + +export async function upsertPackagistVersions( + qx: QueryExecutor, + packageId: string, + versions: PackagistVersionInput[], + latestNumber: string | null, +): Promise<{ changedFields: string[]; versionIds: Array<{ number: string; id: string }> }> { + if (versions.length === 0) return { changedFields: [], versionIds: [] } + + const row: { + changed_fields: string[] + version_ids: Array<{ number: string; id: string }> + } = await qx.selectOne( + `WITH old AS ( + SELECT number, published_at, is_latest, is_prerelease, licenses + FROM versions + WHERE package_id = $(packageId)::bigint AND number = ANY($(numbers)::text[]) + ), + ins AS ( + INSERT INTO versions ( + package_id, ecosystem, namespace, name, number, + published_at, is_latest, is_prerelease, licenses, last_synced_at, + created_at + ) + SELECT $(packageId)::bigint, 'packagist', p.namespace, p.name, v.num, + v.pub::timestamptz, v.latest, v.pre, + CASE WHEN v.lic IS NULL THEN NULL::text[] ELSE ARRAY[v.lic] END, + NOW(), NOW() + FROM unnest( + $(numbers)::text[], + $(publishedAts)::text[], + $(isLatests)::bool[], + $(isPrereleases)::bool[], + $(licenses)::text[] + ) AS v(num, pub, latest, pre, lic) + CROSS JOIN (SELECT namespace, name FROM packages WHERE id = $(packageId)::bigint) p + ON CONFLICT (package_id, number) DO UPDATE SET + published_at = COALESCE(EXCLUDED.published_at, versions.published_at), + is_latest = EXCLUDED.is_latest, + is_prerelease = EXCLUDED.is_prerelease, + licenses = EXCLUDED.licenses, + last_synced_at = EXCLUDED.last_synced_at + RETURNING number, published_at, is_latest, is_prerelease, licenses, id::text AS id + ) + SELECT array_remove(ARRAY[ + CASE WHEN bool_or(o.number IS NULL) THEN 'versions.number' END, + CASE WHEN bool_or(o.number IS NULL OR o.published_at IS DISTINCT FROM ins.published_at) THEN 'versions.published_at' END, + CASE WHEN bool_or(o.is_latest IS DISTINCT FROM ins.is_latest) THEN 'versions.is_latest' END, + CASE WHEN bool_or(o.is_prerelease IS DISTINCT FROM ins.is_prerelease) THEN 'versions.is_prerelease' END, + CASE WHEN bool_or(o.licenses IS DISTINCT FROM ins.licenses) THEN 'versions.licenses' END + ], NULL) AS changed_fields, + json_agg(json_build_object('number', ins.number, 'id', ins.id))::jsonb AS version_ids + FROM ins LEFT JOIN old o ON o.number = ins.number`, + { + packageId, + numbers: versions.map((v) => v.number), + publishedAts: versions.map((v) => v.publishedAt), + isLatests: versions.map((v) => v.isLatest), + isPrereleases: versions.map((v) => v.isPrerelease), + licenses: versions.map((v) => v.license), + }, + ) + + // Parse the version_ids from JSONB + const versionIds = (row.version_ids as Array<{ number: string; id: string }>) || [] + + // Clear a stale is_latest on every OTHER version of this package. Anchored on the declared + // latest (latestNumber) — NOT on what's in this batch — so a previously-latest version whose + // records were updated differently can't keep is_latest = true alongside the new latest. When + // no latest is known, leave flags untouched rather than wipe all. + if (latestNumber != null) { + await qx.result( + `UPDATE versions SET is_latest = false + WHERE package_id = $(packageId)::bigint + AND is_latest = true + AND number <> $(latestNumber)`, + { packageId, latestNumber }, + ) + } + + return { changedFields: row.changed_fields, versionIds } +} + export interface PypiVersionInput { number: string publishedAt: string | null From 6394fcef9856a26036a2b42a58deac0fe360cd65 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 14:03:39 +0200 Subject: [PATCH 02/56] fix: race ipv4/ipv6 on packagist fetch connects Signed-off-by: anilb --- .../src/packagist/fetchPackage.ts | 23 +++++++++++++++---- .../src/packagist/listPackages.ts | 10 +++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/fetchPackage.ts b/services/apps/packages_worker/src/packagist/fetchPackage.ts index f791de9c90..1ddcc76ff9 100644 --- a/services/apps/packages_worker/src/packagist/fetchPackage.ts +++ b/services/apps/packages_worker/src/packagist/fetchPackage.ts @@ -1,3 +1,5 @@ +import { Agent, type Dispatcher } from 'undici' + import type { FetchError, P2Metadata, @@ -6,6 +8,14 @@ import type { PackagistStatsJson, } from './types' +// Shared dispatcher for all packagist fetches. autoSelectFamily races IPv4/IPv6 +// (Happy Eyeballs) — without it, an environment with broken IPv6 (common in +// containers) hangs on the first resolved AAAA address until UND_ERR_CONNECT_TIMEOUT. +// Connect timeout stays modest; the per-request 30s abort still bounds the whole call. +export const packagistDispatcher: Dispatcher = new Agent({ + connect: { timeout: 15_000, autoSelectFamily: true }, +}) + export function buildPackagistUserAgent(): string { const mailto = process.env.CROWD_PACKAGES_PACKAGIST_MAILTO || 'oss-packages@linuxfoundation.org' return `lfx-packages-worker/0.1 (+https://lfx.linuxfoundation.org; mailto=${mailto})` @@ -27,13 +37,16 @@ export async function fetchPackagistStats(name: string): Promise try { let res: Response try { - res = await fetch(PACKAGIST_LIST, { + const init: RequestInit & { dispatcher?: Dispatcher } = { headers: { Accept: 'application/json', 'User-Agent': buildPackagistUserAgent(), }, signal: abort.signal, - }) + dispatcher: packagistDispatcher, + } + res = await fetch(PACKAGIST_LIST, init as RequestInit) } catch (err) { return { kind: 'TRANSIENT', message: describeFetchFailure(err) } } From 4cb45280e4dc33b8e2a6ba5e09005e562e6c8c9f Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 14:07:54 +0200 Subject: [PATCH 03/56] fix: cap packagist connection pool at crawl concurrency Signed-off-by: anilb --- services/apps/packages_worker/src/packagist/fetchPackage.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/apps/packages_worker/src/packagist/fetchPackage.ts b/services/apps/packages_worker/src/packagist/fetchPackage.ts index 1ddcc76ff9..266c11d454 100644 --- a/services/apps/packages_worker/src/packagist/fetchPackage.ts +++ b/services/apps/packages_worker/src/packagist/fetchPackage.ts @@ -12,8 +12,12 @@ import type { // (Happy Eyeballs) — without it, an environment with broken IPv6 (common in // containers) hangs on the first resolved AAAA address until UND_ERR_CONNECT_TIMEOUT. // Connect timeout stays modest; the per-request 30s abort still bounds the whole call. +// `connections` caps sockets per origin at the crawl concurrency, so bursts queue on +// warm keep-alive connections instead of opening fresh ones — connection setup is +// where the intermittent UND_ERR_CONNECT_TIMEOUTs were observed under load. export const packagistDispatcher: Dispatcher = new Agent({ connect: { timeout: 15_000, autoSelectFamily: true }, + connections: 10, }) export function buildPackagistUserAgent(): string { From 0ba8558dad3f67ef59397c09afc8adc414a6450e Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 17:38:05 +0200 Subject: [PATCH 04/56] fix: strip NUL bytes from packagist registry data Signed-off-by: anilb --- .../__tests__/persistMetadata.test.ts | 15 +++++++++++++++ .../__tests__/persistPackageInfo.test.ts | Bin 4963 -> 5422 bytes .../src/packagist/upsertMetadata.ts | 6 ++++++ .../src/packagist/upsertPackageInfo.ts | 5 +++++ 4 files changed, 26 insertions(+) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts index df9dbc02ba..8ada433886 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts @@ -162,4 +162,19 @@ describe('persistPackagistMetadata', () => { expect(mockDeps).not.toHaveBeenCalled() expect(result.found).toBe(true) }) + + it('strips NUL bytes from the homepage before writing (Postgres rejects them)', async () => { + mockAggregates.mockResolvedValue({ id: '9', changedFields: [] }) + mockVersions.mockResolvedValue({ changedFields: [], versionIds: [] }) + + await persistPackagistMetadata(qx, PURL, [ + { version: '1.0.0', version_normalized: '1.0.0.0', homepage: 'https://ex\0ample.org' }, + ]) + + expect(mockAggregates).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ homepage: 'https://example.org' }), + ) + }) }) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts index 0d90227b7c720401c9b6a033b226cf0b27eddfa6..e2d87122b231649c422164f5f72eb4466a6d472b 100644 GIT binary patch delta 245 zcmaE?woYq9f^aaGf9q3l2}r#1LY$bX{Dg<3Y1d-no*jU2(v6XlOa!`C^0W3KUW>1sdn;B_DBbqJjgM) Ubiy?%)M`TXY67jT)#L&q0O1Qyr2qf` delta 11 ScmZ3d^;m5~f-rNfCKmu3q64@9 diff --git a/services/apps/packages_worker/src/packagist/upsertMetadata.ts b/services/apps/packages_worker/src/packagist/upsertMetadata.ts index 2f7520ab7c..d14e25d2eb 100644 --- a/services/apps/packages_worker/src/packagist/upsertMetadata.ts +++ b/services/apps/packages_worker/src/packagist/upsertMetadata.ts @@ -7,6 +7,8 @@ import { import type { VersionDependencyEdge } from '@crowd/data-access-layer/src/packages' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { stripNullBytesDeep } from '../utils/stripNullBytesDeep' + import { buildPackagistVersionRows, extractVersionDependencies, @@ -19,6 +21,10 @@ export async function persistPackagistMetadata( purl: string, expanded: PackagistExpandedVersion[], ): Promise<{ found: boolean; changedFields: string[]; unresolvedDependencyTargets: number }> { + // Registry data can contain NUL bytes (e.g. mojibake descriptions/licenses) that + // Postgres text columns reject; strip them before any field is persisted. + stripNullBytesDeep(expanded) + const { versionRows, latestVersion, firstReleaseAt, latestReleaseAt, licenses, homepage } = buildPackagistVersionRows(expanded) diff --git a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts index cc179f9238..2e3c14d5f5 100644 --- a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts +++ b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts @@ -7,6 +7,7 @@ import { import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl' +import { stripNullBytesDeep } from '../utils/stripNullBytesDeep' import type { NormalizedPackagistStats } from './types' @@ -18,6 +19,10 @@ export async function persistPackagistPackageInfo( purl: string, stats: NormalizedPackagistStats, ): Promise<{ found: boolean; changedFields: string[] }> { + // Registry data can contain NUL bytes (e.g. mojibake descriptions) that Postgres + // text columns reject; strip them before any field is persisted. + stripNullBytesDeep(stats) + // Step 1: Update packages row const canonical = stats.repositoryUrl ? canonicalizeRepoUrl(stats.repositoryUrl) : null const result = await updatePackagistPackageStats(qx, { From 739cae73ab293fbe2fda95af5bc1751a39da1975 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 18:36:47 +0200 Subject: [PATCH 05/56] fix: run packagist daily downloads later in UTC day Signed-off-by: anilb --- services/apps/packages_worker/src/packagist/schedule.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/apps/packages_worker/src/packagist/schedule.ts b/services/apps/packages_worker/src/packagist/schedule.ts index 9cc646792b..e2556f0d66 100644 --- a/services/apps/packages_worker/src/packagist/schedule.ts +++ b/services/apps/packages_worker/src/packagist/schedule.ts @@ -7,7 +7,12 @@ import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/clien export const PACKAGIST_CRONS = { seed: '17 2 * * 0', downloads30d: '53 3 1 * *', - downloadsDaily: '23 6 * * *', + // Late in the UTC day on purpose: Packagist's `daily` figure is `today_so_far + + // yesterday_total * dayRatio` (DownloadManager.php), where dayRatio shrinks toward 0 + // as the day progresses — querying at 22:23 UTC (dayRatio ~0.08) captures a figure + // that's ~92% today's real data instead of ~74% borrowed from yesterday at the old + // 06:23 slot, while keeping a ~1h37m buffer before UTC midnight for the drain to finish. + downloadsDaily: '23 22 * * *', } // Workflow types by name (not function reference) so this module doesn't pull the From 1c9596b57f2082b28687800017b49b7ef93cb021 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 18:40:51 +0200 Subject: [PATCH 06/56] fix: pin packagist daily downloads runDate to run cutoff Signed-off-by: anilb --- services/apps/packages_worker/src/packagist/activities.ts | 2 +- services/apps/packages_worker/src/packagist/workflows.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index 7499f65a63..1331c6f2cf 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -364,10 +364,10 @@ export async function getPackagistDailyBatch( export async function ingestPackagistDailyBatch( candidates: PackagistDailyCandidate[], + runDate: string, ): Promise { if (candidates.length === 0) return const qx = await getPackagesDb() - const runDate = new Date().toISOString().slice(0, 10) const attempt = Context.current().info.attempt await ingestPackagistItemsConcurrently( diff --git a/services/apps/packages_worker/src/packagist/workflows.ts b/services/apps/packages_worker/src/packagist/workflows.ts index 9225126ef6..d0e1bf64c9 100644 --- a/services/apps/packages_worker/src/packagist/workflows.ts +++ b/services/apps/packages_worker/src/packagist/workflows.ts @@ -75,12 +75,16 @@ export async function ingestPackagistDownloadsDaily(state: DownloadsState = {}): if ((await acts.getCriticalPackagistCount()) === 0) return const cutoff = state.cutoff ?? (await acts.packagistCurrentTimestamp()) + // Packagist's `daily` figure is tied to a specific calendar day (see schedule.ts) — + // derive the write-date from the run's fixed cutoff instead of re-reading the clock + // per batch, so a drain that runs past UTC midnight still tags every row consistently. + const runDate = cutoff.slice(0, 10) const stopAfterFirstPage = await acts.packagistStopAfterFirstPage() for (let r = 0; r < ROUNDS_PER_RUN; r++) { const candidates = await acts.getPackagistDailyBatch(cutoff, INGEST_BATCH) if (candidates.length === 0) return - await acts.ingestPackagistDailyBatch(candidates) + await acts.ingestPackagistDailyBatch(candidates, runDate) if (stopAfterFirstPage) return if (candidates.length < INGEST_BATCH) return } From 634d1f7f8c9cfa0c76c4feb5357cd709a3cb6038 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 18:57:56 +0200 Subject: [PATCH 07/56] docs: note packagist 30d drain month-boundary gap Signed-off-by: anilb --- services/apps/packages_worker/src/packagist/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index fc17d11d26..d63935a820 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -129,6 +129,14 @@ never overwritten — the first observation (closest to the boundary) wins. Packagist keeps no history, so unlike npm there is no backfill lane; a missed month stays missed. State: `downloads_30d_last_run_at` + `downloads_30d_run_result`. +`monthlyWindowFor()` labels `end_date` from whatever *month* the batch executes +in — not the exact day — so a drain that runs long but stays inside the same +month is still labeled consistently. The one gap: if the full-catalog drain +ever dragged on long enough to spill past the next month's 1st, later batches +would get mislabeled with the new month's `end_date` instead of the run's. +Unlike the daily lane (see below), this isn't pinned to a fixed run cutoff — +low risk given the run should finish in hours, not weeks, but worth knowing. + --- ## 4. `ingestPackagistDownloadsDaily` — daily capture (critical slice) From f43c90333f97ce1cb424a3db25ce52f705132948 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 19:04:13 +0200 Subject: [PATCH 08/56] docs: fix stale packagist daily schedule time, note drift Signed-off-by: anilb --- .../packages_worker/src/packagist/README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index d63935a820..d7ffcec615 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -141,7 +141,11 @@ low risk given the run should finish in hours, not weeks, but worth knowing. ## 4. `ingestPackagistDownloadsDaily` — daily capture (critical slice) -**Schedule:** `packagist-downloads-daily`, daily `06:23` UTC. +**Schedule:** `packagist-downloads-daily`, daily `22:23` UTC — late in the day +on purpose: Packagist's `daily` figure is `today_so_far + yesterday_total × +dayRatio`, with `dayRatio` decaying toward 0 as the day goes on, so asking +late gets a figure that's mostly today's real data instead of mostly +borrowed from yesterday. **Targets:** `is_critical = TRUE` only; no-ops (guard) while zero packagist rows are critical — i.e. until the first `rank_packages()` pass. Same cutoff watermark pattern as the 30d lane. @@ -149,9 +153,14 @@ watermark pattern as the 30d lane. **What it does:** fetches the dynamic endpoint per critical package and records the registry's rolling daily figure. -**Populates:** one `downloads_daily` row per package per day -(`package_id`, run date, count). State: `daily_downloads_last_run_at` + -`daily_downloads_run_result`. +**Populates:** one `downloads_daily` row per package per day (`package_id`, +run date, count), with the run date pinned once from the run's `cutoff` so +every row from one run shares the same label even if the drain runs long. +That only fixes the *label*, though — Packagist computes `daily` live, +relative to the moment each package is actually fetched, so a drain that +runs past real UTC midnight still hands back a value already drifted toward +the new day for whichever packages are processed last, stored under the +older date. State: `daily_downloads_last_run_at` + `daily_downloads_run_result`. --- From 188728d758f6ba8f49b97e1a0ec248d1b73e23ea Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 19:10:43 +0200 Subject: [PATCH 09/56] docs: correct packagist rate-limit and 404-retry claims Signed-off-by: anilb --- .../packages_worker/src/packagist/README.md | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index d7ffcec615..302d3c7fe1 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -20,25 +20,31 @@ Shared design notes: (`repo.packagist.org/p2/{vendor}/{name}.json`, CDN-served, the same file Composer clients resolve) carries the version manifests: every tagged release with its `require`/`require-dev`, licenses, homepage. -- **Politeness.** Bounded concurrency per Packagist's guidelines — max 10 - in-flight dynamic requests (their published limit; running hotter gets - connections reset). Since every lane's ingest starts with a dynamic fetch, - all lanes run at that cap (env-tunable downward, hard-capped at 10); the - p2 fetches inside the metadata lane inherit it, well under p2's own 20. - User-Agent carries a `mailto=` contact - (`CROWD_PACKAGES_PACKAGIST_MAILTO`). Cron minutes are deliberately off `:00`. - p2 fetches replay the stored `Last-Modified` as `If-Modified-Since`; a `304` - records the run without re-downloading or re-parsing. +- **Politeness.** Bounded concurrency per Packagist's own published etiquette + guideline ([packagist.org/apidoc](https://packagist.org/apidoc)): max 10 + concurrent requests, 20 if static-file-only. Their docs state this as "be a + good citizen," not a documented enforcement mechanism — we don't have a + citation for what happens above it, so treat 10 as the number to respect, + not a proven "or else". Since every lane's ingest starts with a dynamic + fetch, all lanes run at that stricter cap anyway (env-tunable downward, + hard-capped at 10); the p2 fetches inside the metadata lane inherit it, + well under p2's own 20. User-Agent carries a `mailto=` contact + (`CROWD_PACKAGES_PACKAGIST_MAILTO`) — their docs say a missing one risks an + IP block. Cron minutes are deliberately off `:00` (their docs also flag + hourly traffic peaks at `XX:00`). p2 fetches replay the stored + `Last-Modified` as `If-Modified-Since`; a `304` records the run without + re-downloading or re-parsing. - **Watermark-driven drains.** Each lane tracks its own per-purl watermark + run result in `packagist_package_state` (`metadata_last_run_at` / `downloads_30d_last_run_at` / `daily_downloads_last_run_at`, each with a `*_run_result` JSONB). Workflows drain in `continueAsNew` rounds (20 × 50), so an interrupted run resumes where it left off on the next launch. -- **Error handling.** 404 → mark the lane's watermark with an error result, - skip, don't retry. Malformed body/shape → 3 fast in-lane retries (1s linear - backoff), then mark errored and move on. 429/5xx/network/timeout → throw and - ride Temporal's exponential activity retry (5 attempts, lockstep with the - per-item give-up so one bad package can never stall a drain). +- **Error handling.** 404 or malformed body/shape → 3 fast in-lane retries + (1s linear backoff), then mark the lane's watermark with an error result and + move on — `isClientError()` treats 404 the same as any other non-429 4xx, so + there's no zero-retry fast path for it specifically. 429/5xx/network/timeout + → throw and ride Temporal's exponential activity retry (5 attempts, lockstep + with the per-item give-up so one bad package can never stall a drain). - **Tier scoping.** Discovery, package info, repo links, versions, and dependencies cover **all** packages (deliberate divergence from pypi — see ADR-0006). Only `downloads_daily` and maintainers are From 73ba79eb79f26af823154a525453f58efef94987 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 19:12:23 +0200 Subject: [PATCH 10/56] docs: simplify packagist drift notes Signed-off-by: anilb --- .../packages_worker/src/packagist/README.md | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index 302d3c7fe1..f920592f1a 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -135,13 +135,10 @@ never overwritten — the first observation (closest to the boundary) wins. Packagist keeps no history, so unlike npm there is no backfill lane; a missed month stays missed. State: `downloads_30d_last_run_at` + `downloads_30d_run_result`. -`monthlyWindowFor()` labels `end_date` from whatever *month* the batch executes -in — not the exact day — so a drain that runs long but stays inside the same -month is still labeled consistently. The one gap: if the full-catalog drain -ever dragged on long enough to spill past the next month's 1st, later batches -would get mislabeled with the new month's `end_date` instead of the run's. -Unlike the daily lane (see below), this isn't pinned to a fixed run cutoff — -low risk given the run should finish in hours, not weeks, but worth knowing. +Unlike the daily lane (see below), the write-date here isn't pinned to a fixed +run cutoff — if the full-catalog drain ran long enough, the labeled window +could drift forward from what was true when the run started. Low risk given +the run should finish in hours, not weeks, but worth knowing. --- @@ -162,11 +159,10 @@ the registry's rolling daily figure. **Populates:** one `downloads_daily` row per package per day (`package_id`, run date, count), with the run date pinned once from the run's `cutoff` so every row from one run shares the same label even if the drain runs long. -That only fixes the *label*, though — Packagist computes `daily` live, -relative to the moment each package is actually fetched, so a drain that -runs past real UTC midnight still hands back a value already drifted toward -the new day for whichever packages are processed last, stored under the -older date. State: `daily_downloads_last_run_at` + `daily_downloads_run_result`. +That only fixes the *label*, though — Packagist computes `daily` live at +fetch time, so for a long-running drain the underlying value can drift +forward from what the label implies for whichever packages are processed +last. State: `daily_downloads_last_run_at` + `daily_downloads_run_result`. --- From f86865fe4c38b232ec8a37e6af4fe3396d17e6c3 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 19:15:52 +0200 Subject: [PATCH 11/56] fix: use public DNS for packagist worker Signed-off-by: anilb --- scripts/services/packagist-worker.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/services/packagist-worker.yaml b/scripts/services/packagist-worker.yaml index 3956a8c51a..8e7ea99b8c 100644 --- a/scripts/services/packagist-worker.yaml +++ b/scripts/services/packagist-worker.yaml @@ -41,6 +41,9 @@ services: - ../../backend/.env.override.composed environment: <<: *env-args + dns: + - 8.8.8.8 + - 1.1.1.1 hostname: packagist-worker networks: - crowd-bridge From 685f8b0c2534930595e77d49d743d419796ee489 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 19:18:07 +0200 Subject: [PATCH 12/56] docs: correct ADR decider full name Signed-off-by: anilb --- docs/adr/0005-pypi-downloads-bigquery-merge-scoping.md | 2 +- docs/adr/0006-packagist-worker-design-decisions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/adr/0005-pypi-downloads-bigquery-merge-scoping.md b/docs/adr/0005-pypi-downloads-bigquery-merge-scoping.md index 6089d04062..fc2d530671 100644 --- a/docs/adr/0005-pypi-downloads-bigquery-merge-scoping.md +++ b/docs/adr/0005-pypi-downloads-bigquery-merge-scoping.md @@ -2,7 +2,7 @@ **Date**: 2026-07-01 **Status**: accepted -**Deciders**: Anil B +**Deciders**: Anil Bostanci _Consolidated ADR for the PyPI downloads worker — record further PyPI-worker download decisions here rather than opening new ADRs._ diff --git a/docs/adr/0006-packagist-worker-design-decisions.md b/docs/adr/0006-packagist-worker-design-decisions.md index c21a9facb2..02e0462e04 100644 --- a/docs/adr/0006-packagist-worker-design-decisions.md +++ b/docs/adr/0006-packagist-worker-design-decisions.md @@ -2,7 +2,7 @@ **Date**: 2026-07-13 **Status**: living -**Deciders**: Anil B +**Deciders**: Anil Bostanci _Living, consolidated record for the Packagist (PHP/Composer) worker. Record further Packagist-worker decisions here as new `### subsections` under Decisions rather than opening a From 820e909afb3d701d5e9b3cc8d4e6320ab752ef2a Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 19:21:30 +0200 Subject: [PATCH 13/56] docs: spell out full packagist env var names Signed-off-by: anilb --- .../apps/packages_worker/src/packagist/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index f920592f1a..47158e1452 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -182,13 +182,13 @@ last. State: `daily_downloads_last_run_at` + `daily_downloads_run_result`. All optional; defaults in `activities.ts` / `fetchPackage.ts`: -| Env var (`CROWD_PACKAGES_PACKAGIST_…`) | Default | Meaning | +| Env var | Default | Meaning | |---|---|---| -| `MAILTO` | `oss-packages@linuxfoundation.org` | Contact in the crawl User-Agent | -| `STATS_CONCURRENCY` | 10 (hard cap 10) | Parallel package ingests across all lanes — every lane starts with a dynamic-endpoint fetch, so the stricter 10-limit bounds everything | -| `METADATA_REFRESH_DAYS` | 7 | Metadata refresh window | -| `RUN_ONLY_FOR_CRITICAL` | `false` | `true` narrows metadata to the critical slice | -| `STOP_AFTER_FIRST_PAGE` | `false` | Debug: one drain page, no continueAsNew | +| `CROWD_PACKAGES_PACKAGIST_MAILTO` | `oss-packages@linuxfoundation.org` | Contact in the crawl User-Agent | +| `CROWD_PACKAGES_PACKAGIST_STATS_CONCURRENCY` | 10 (hard cap 10) | Parallel package ingests across all lanes — every lane starts with a dynamic-endpoint fetch, so the stricter 10-limit bounds everything | +| `CROWD_PACKAGES_PACKAGIST_METADATA_REFRESH_DAYS` | 7 | Metadata refresh window | +| `CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL` | `false` | `true` narrows metadata to the critical slice | +| `CROWD_PACKAGES_PACKAGIST_STOP_AFTER_FIRST_PAGE` | `false` | Debug: one drain page, no continueAsNew | ## Running it From b3daa48bb9a8d767a60c41e4f6277d005f076275 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 19:31:11 +0200 Subject: [PATCH 14/56] docs: renumber packagist ADR to 0009 Signed-off-by: anilb --- ...n-decisions.md => 0009-packagist-worker-design-decisions.md} | 0 services/apps/packages_worker/src/packagist/README.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/adr/{0006-packagist-worker-design-decisions.md => 0009-packagist-worker-design-decisions.md} (100%) diff --git a/docs/adr/0006-packagist-worker-design-decisions.md b/docs/adr/0009-packagist-worker-design-decisions.md similarity index 100% rename from docs/adr/0006-packagist-worker-design-decisions.md rename to docs/adr/0009-packagist-worker-design-decisions.md diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index 47158e1452..4e167f2e69 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -209,4 +209,4 @@ Note: triggering `seed` also chain-starts the full `metadata` drain (set Local smoke order: seed → metadata → (rank) → downloads lanes for critical-scoped writes. State lives in `packagist_package_state` (migration `V1783382400__packagist_worker.sql`); design decisions in -`docs/adr/0006-packagist-worker-design-decisions.md`. +`docs/adr/0009-packagist-worker-design-decisions.md`. From 4c26f3c0d2c6810d54c0b3fd9d64b055c30fdf62 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 19:42:58 +0200 Subject: [PATCH 15/56] fix: avoid ReDoS in packagist name validation Signed-off-by: anilb --- .../src/packagist/__tests__/listPackages.test.ts | 11 +++++++++++ .../packages_worker/src/packagist/listPackages.ts | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts b/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts index 31d39ebab0..0862097d66 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts @@ -47,6 +47,17 @@ describe('parsePackagistPackageList', () => { expect(invalid).toBe(7) }) + it('rejects a long invalid segment without catastrophic regex backtracking', () => { + // CodeQL js/redos: a long run of one char class followed by a char that breaks + // the match is the classic trigger for exponential backtracking. + const pathological = 'vendor/' + '0'.repeat(50) + '!' + const start = Date.now() + const { entries, invalid } = parsePackagistPackageList({ packageNames: [pathological] }) + expect(Date.now() - start).toBeLessThan(200) + expect(entries).toEqual([]) + expect(invalid).toBe(1) + }) + it('lowercases and deduplicates names without counting duplicates as invalid', () => { const { entries, invalid } = parsePackagistPackageList({ packageNames: ['Monolog/Monolog', 'monolog/monolog'], diff --git a/services/apps/packages_worker/src/packagist/listPackages.ts b/services/apps/packages_worker/src/packagist/listPackages.ts index ed14b3064a..4b3e099af9 100644 --- a/services/apps/packages_worker/src/packagist/listPackages.ts +++ b/services/apps/packages_worker/src/packagist/listPackages.ts @@ -4,7 +4,10 @@ import { buildPackagistUserAgent, describeFetchFailure, packagistDispatcher } fr import type { FetchError } from './types' const PACKAGIST_LIST = 'https://packagist.org/packages/list.json' -const COMPOSER_NAME_REGEX = /^[a-z0-9]([_.-]?[a-z0-9]+)*$/ +// Non-backtracking form: a mandatory separator per repetition (vs. an optional one) +// removes the ambiguity that let the old `([_.-]?[a-z0-9]+)*` pattern backtrack +// exponentially on a long run of the same character class (CodeQL js/redos). +const COMPOSER_NAME_REGEX = /^[a-z0-9]+(?:[_.-][a-z0-9]+)*$/ export interface PackagistListEntry { vendor: string From ca72deab6bac8b192e26b75d1b550168c3313f8d Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 16 Jul 2026 19:43:00 +0200 Subject: [PATCH 16/56] fix: guard packagist metadata drain against overlap Signed-off-by: anilb --- .../src/packagist/workflows.ts | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/workflows.ts b/services/apps/packages_worker/src/packagist/workflows.ts index d0e1bf64c9..c84fa215bb 100644 --- a/services/apps/packages_worker/src/packagist/workflows.ts +++ b/services/apps/packages_worker/src/packagist/workflows.ts @@ -1,4 +1,11 @@ -import { ParentClosePolicy, continueAsNew, proxyActivities, startChild } from '@temporalio/workflow' +import { + ParentClosePolicy, + WorkflowIdReusePolicy, + continueAsNew, + log, + proxyActivities, + startChild, +} from '@temporalio/workflow' import type * as activities from './activities' import { INGEST_MAX_ATTEMPTS } from './retryPolicy' @@ -30,10 +37,25 @@ export async function seedPackagistPackages(): Promise { // discovered packages are rows before the p2 crawl resolves dependency targets. // Started (not awaited) with ABANDON — the drain runs for hours and must outlive // this workflow; the state watermarks make the pass self-advancing regardless. - await startChild(ingestPackagistMetadata, { - args: [{}], - parentClosePolicy: ParentClosePolicy.ABANDON, - }) + // Fixed workflowId + ALLOW_DUPLICATE: starting a duplicate against a still-RUNNING + // execution with the same id always throws WorkflowExecutionAlreadyStartedError + // regardless of reuse policy, so a drain that outlasts the week makes next + // Sunday's seed skip its chain-start instead of doubling the crawl; ALLOW_DUPLICATE + // just lets the id be reused once that prior execution has actually closed. + try { + await startChild(ingestPackagistMetadata, { + workflowId: 'packagist-metadata-drain', + workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE, + args: [{}], + parentClosePolicy: ParentClosePolicy.ABANDON, + }) + } catch (err) { + if (err instanceof Error && err.name === 'WorkflowExecutionAlreadyStartedError') { + log.warn('packagist metadata drain still running from a prior seed — skipping chain-start') + return + } + throw err + } } export async function ingestPackagistMetadata(state: MetadataState = {}): Promise { From 8ffb3cf596a92ba08d8a82faec2429b8da70b45d Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 10:53:36 +0200 Subject: [PATCH 17/56] fix: preserve full license array per packagist version Signed-off-by: anilb --- .../src/packagist/__tests__/normalize.test.ts | 4 ++-- .../packages_worker/src/packagist/normalize.ts | 2 +- .../apps/packages_worker/src/packagist/types.ts | 2 +- .../data-access-layer/src/packages/versions.ts | 14 +++++++++++--- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts index 511a8916cf..02169f5cb6 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts @@ -139,10 +139,10 @@ describe('buildPackagistVersionRows', () => { publishedAt: '2024-03-01T00:00:00+00:00', isLatest: true, isPrerelease: false, - license: 'MIT', + licenses: ['MIT'], }) expect(byNumber['2.9.1'].isLatest).toBe(false) - expect(byNumber['1.0.0'].license).toBe('BSD-3-Clause') + expect(byNumber['1.0.0'].licenses).toEqual(['BSD-3-Clause']) expect(latestVersion).toBe('2.10.0') expect(firstReleaseAt).toBe('2020-06-01T00:00:00+00:00') expect(latestReleaseAt).toBe('2024-03-01T00:00:00+00:00') diff --git a/services/apps/packages_worker/src/packagist/normalize.ts b/services/apps/packages_worker/src/packagist/normalize.ts index c73583191d..642e9079f7 100644 --- a/services/apps/packages_worker/src/packagist/normalize.ts +++ b/services/apps/packages_worker/src/packagist/normalize.ts @@ -146,7 +146,7 @@ export function buildPackagistVersionRows(versions: PackagistExpandedVersion[]): publishedAt: v.time ?? null, isLatest: false, // Will be set below isPrerelease: isPackagistPrerelease(v.version_normalized ?? v.version), - license: (v.license && v.license.length > 0 ? v.license[0] : null) ?? null, + licenses: v.license && v.license.length > 0 ? v.license : null, })) // Find latest: prefer stable over prerelease; within each group use Composer ordering diff --git a/services/apps/packages_worker/src/packagist/types.ts b/services/apps/packages_worker/src/packagist/types.ts index dbb7baa360..d514044e98 100644 --- a/services/apps/packages_worker/src/packagist/types.ts +++ b/services/apps/packages_worker/src/packagist/types.ts @@ -105,7 +105,7 @@ export interface PackagistVersionRow { publishedAt: string | null isLatest: boolean isPrerelease: boolean - license: string | null + licenses: string[] | null } export interface PackagistDependency { diff --git a/services/libs/data-access-layer/src/packages/versions.ts b/services/libs/data-access-layer/src/packages/versions.ts index c2d08e0784..2a127dabd7 100644 --- a/services/libs/data-access-layer/src/packages/versions.ts +++ b/services/libs/data-access-layer/src/packages/versions.ts @@ -74,7 +74,9 @@ export interface PackagistVersionInput { publishedAt: string | null isLatest: boolean isPrerelease: boolean - license: string | null + // Composer allows dual/multi-licensed releases (e.g. ["MIT", "Apache-2.0"]) — the + // full array is preserved, unlike the single-SPDX-string npm/pypi license inputs. + licenses: string[] | null } export async function upsertPackagistVersions( @@ -102,7 +104,11 @@ export async function upsertPackagistVersions( ) SELECT $(packageId)::bigint, 'packagist', p.namespace, p.name, v.num, v.pub::timestamptz, v.latest, v.pre, - CASE WHEN v.lic IS NULL THEN NULL::text[] ELSE ARRAY[v.lic] END, + -- licenses travels as one JSON-encoded array per row (unnest can't carry a + -- ragged array-of-arrays alongside scalar columns), decoded back here. + CASE WHEN v.lic IS NULL THEN NULL::text[] + ELSE (SELECT array_agg(elem) FROM jsonb_array_elements_text(v.lic::jsonb) AS elem) + END, NOW(), NOW() FROM unnest( $(numbers)::text[], @@ -135,7 +141,9 @@ export async function upsertPackagistVersions( publishedAts: versions.map((v) => v.publishedAt), isLatests: versions.map((v) => v.isLatest), isPrereleases: versions.map((v) => v.isPrerelease), - licenses: versions.map((v) => v.license), + licenses: versions.map((v) => + v.licenses && v.licenses.length > 0 ? JSON.stringify(v.licenses) : null, + ), }, ) From 398f4be7e5dbaa7df897a74998ed7a506038b07a Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 10:54:20 +0200 Subject: [PATCH 18/56] fix: bump last_synced_at on packagist 30d mirror Signed-off-by: anilb --- .../libs/data-access-layer/src/packages/downloadsLast30d.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/libs/data-access-layer/src/packages/downloadsLast30d.ts b/services/libs/data-access-layer/src/packages/downloadsLast30d.ts index 8ac1e3e718..f27e746af3 100644 --- a/services/libs/data-access-layer/src/packages/downloadsLast30d.ts +++ b/services/libs/data-access-layer/src/packages/downloadsLast30d.ts @@ -163,9 +163,12 @@ export async function upsertLast30dDownload( ) const changed = row.changed_fields if (mirrorToPackages) { + // last_synced_at is the Tinybird ENGINE_VER for the packages datasource — it must + // move whenever a real column changes, or Tinybird can keep serving a stale row. const rowCount = await qx.result( `UPDATE packages - SET downloads_last_30d = $(count) + SET downloads_last_30d = $(count), + last_synced_at = NOW() WHERE purl = $(purl) AND downloads_last_30d IS DISTINCT FROM $(count)`, { count, purl }, ) From d0596c898ddd062fb8e9b36511bdb226ff441f19 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 11:00:30 +0200 Subject: [PATCH 19/56] fix: audit all packagist changed-fields consistently Signed-off-by: anilb --- .../0009-packagist-worker-design-decisions.md | 16 +++--- docs/adr/README.md | 2 +- .../src/packagist/__tests__/downloads.test.ts | 17 ++++--- .../src/packagist/__tests__/ingest.test.ts | 36 +++++++++----- .../__tests__/persistMetadata.test.ts | 7 ++- .../__tests__/persistPackageInfo.test.ts | Bin 5422 -> 5645 bytes .../src/packagist/activities.ts | 23 ++++++--- .../src/packagist/downloads.ts | 10 ++-- .../src/packagist/upsertMetadata.ts | 3 +- .../src/packagist/upsertPackageInfo.ts | 3 +- .../src/packages/dependencies.ts | 47 +++++++++++++----- 11 files changed, 110 insertions(+), 54 deletions(-) diff --git a/docs/adr/0009-packagist-worker-design-decisions.md b/docs/adr/0009-packagist-worker-design-decisions.md index 02e0462e04..5d77f0b932 100644 --- a/docs/adr/0009-packagist-worker-design-decisions.md +++ b/docs/adr/0009-packagist-worker-design-decisions.md @@ -1,7 +1,7 @@ -# ADR-0006: Packagist worker — design decisions (living) +# ADR-0009: Packagist worker — design decisions (living) **Date**: 2026-07-13 -**Status**: living +**Status**: accepted **Deciders**: Anil Bostanci _Living, consolidated record for the Packagist (PHP/Composer) worker. Record further @@ -38,8 +38,8 @@ We store, in `package_dependencies`, **only the direct dependency edges a versio row per `(version_id, depends_on_id, dependency_kind)` — and we **resolve concrete versions and transitive closure at query time**, never at ingest. -Per stored edge (for the critical slice only — dependency edges ride the same p2 metadata fetch as -versions, which we only run for critical packages): +Per stored edge (for **all** packages — dependency edges ride the same p2 metadata fetch as +versions; see the Metadata enrichment scope decision): - `require` → `dependency_kind = 'direct'`, `require-dev` → `dependency_kind = 'dev'`. - `version_constraint` = the **declared** constraint string verbatim (e.g. `^3.0 || ^2.0`). @@ -75,8 +75,7 @@ continuously re-running a Composer resolver for no durable benefit. **Why store `dev` edges when deps.dev doesn't:** deps.dev's resolved graph is runtime-only — every edge lands as `'direct'`, it has no dev/peer notion. Because we read the raw manifest we can split `require-dev` into `'dev'`, so Packagist edges are *richer per-edge* (dev deps + declared -constraints) even though they are *shallower* (no resolved target version) and *narrower* (critical -slice only) than deps.dev's. +constraints) even though they are *shallower* (no resolved target version) than deps.dev's. **Consequences.** @@ -98,8 +97,6 @@ _Negative / trade-offs:_ way there is for npm/maven/pypi/cargo seeded from deps.dev. - Transitive questions ("everything X depends on") pay a recursive walk over direct edges at read time rather than a single indexed lookup. -- Dependency edges exist for the **critical slice only** — non-critical Packagist packages have - none, because the p2 metadata crawl that produces them is critical-scoped. _Risks:_ @@ -198,3 +195,6 @@ that week's enrichment (recoverable via the manual trigger). covers repo linking for all packages and the deliberate critical-only carve-outs). - **2026-07-16** — Added _Lane architecture: seed → chained enrichment, downloads as dedicated lanes_ (removes the stats lane; renames `p2_last_modified` → `metadata_last_modified`). +- **2026-07-17** — Corrected stale "critical slice only" wording left over in the _Dependency + model_ entry from before the _Metadata enrichment scope_ decision widened it to all packages; + set `Status` to `accepted` (matching ADR-0005's precedent for a living/consolidated doc). diff --git a/docs/adr/README.md b/docs/adr/README.md index ca32031ed9..4b8a5da3de 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -12,7 +12,7 @@ Use the `/adr` skill in Claude Code to record new ADRs or query past decisions. | [ADR-0003](./0003-deps-bq-table-selection.md) | Use DependencyGraphEdgesLatest for deps ingestion; defer DependenciesLatest until NUGET or GO needed | accepted | 2026-05-29 | | [ADR-0004](./0004-go-nuget-transitive-dependent-counts.md) | Compute GO/NUGET transitive dependent counts via exact reverse closure (over HLL approximation) | accepted | 2026-06-23 | | [ADR-0005](./0005-pypi-downloads-bigquery-merge-scoping.md) | PyPI downloads via BigQuery bulk export, scoped in the Postgres merge | accepted | 2026-07-01 | -| [ADR-0006](./0006-packagist-worker-design-decisions.md) | Packagist worker — design decisions (living) | living | 2026-07-13 | +| [ADR-0009](./0009-packagist-worker-design-decisions.md) | Packagist worker — design decisions | accepted | 2026-07-13 | ## Why ADRs? diff --git a/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts b/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts index 134031d129..9e10e1db07 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts @@ -50,25 +50,30 @@ describe('monthlyWindowFor', () => { // packages.downloads_last_30d (npm parity). A window already recorded for the month // is never overwritten — the value is the observation closest to the boundary. describe('persistPackagist30dWindow', () => { - it('writes the window with the mirror when the month has no row yet', async () => { - await persistPackagist30dWindow(qx, PURL, 300, '2026-07-01') + it('writes the window with the mirror and returns the changed fields when the month has no row yet', async () => { + mockWindow.mockResolvedValue(['downloads_last_30d.count', 'packages.downloads_last_30d']) + + const changedFields = await persistPackagist30dWindow(qx, PURL, 300, '2026-07-01') expect(mockExistingWindows).toHaveBeenCalledWith(qx, PURL, '2026-07-01', '2026-07-01') expect(mockWindow).toHaveBeenCalledWith(qx, PURL, '2026-06-01', '2026-07-01', 300, true) + expect(changedFields).toEqual(['downloads_last_30d.count', 'packages.downloads_last_30d']) }) - it('does not overwrite an existing window for the month', async () => { + it('does not overwrite an existing window for the month, and returns no changed fields', async () => { mockExistingWindows.mockResolvedValue(['2026-07-01']) - await persistPackagist30dWindow(qx, PURL, 300, '2026-07-15') + const changedFields = await persistPackagist30dWindow(qx, PURL, 300, '2026-07-15') expect(mockWindow).not.toHaveBeenCalled() + expect(changedFields).toEqual([]) }) - it('writes nothing when the registry reported no monthly count', async () => { - await persistPackagist30dWindow(qx, PURL, null, '2026-07-01') + it('writes nothing and returns no changed fields when the registry reported no monthly count', async () => { + const changedFields = await persistPackagist30dWindow(qx, PURL, null, '2026-07-01') expect(mockExistingWindows).not.toHaveBeenCalled() expect(mockWindow).not.toHaveBeenCalled() + expect(changedFields).toEqual([]) }) }) diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts index 94c143404e..3dcd382e72 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -110,7 +110,7 @@ describe('ingestOnePackagistMetadata', () => { }) } - it('persists both endpoints, audits the merged fields, and marks scanned with the fresh Last-Modified', async () => { + it('audits phase 1 immediately and phase 2 separately, then marks scanned with the fresh Last-Modified', async () => { happyMocks() await ingestOnePackagistMetadata(qx, candidate) @@ -119,12 +119,10 @@ describe('ingestOnePackagistMetadata', () => { expect(mockFetchP2).toHaveBeenCalledWith('monolog/monolog', 'Tue, 30 Jun 2026 00:00:00 GMT') expect(mockExpand).toHaveBeenCalledWith(minified) expect(mockPersistMetadata).toHaveBeenCalledWith(qx, PURL, expanded) - expect(mockAudit).toHaveBeenCalledWith( - qx, - 'packagist', - PURL, - expect.arrayContaining(['packages.description', 'versions.number']), - ) + // Two separate audit rows, not one merged call — phase 1 is logged as soon as it's + // committed, before the p2 fetch (which can throw) ever runs. + expect(mockAudit).toHaveBeenNthCalledWith(1, qx, 'packagist', PURL, ['packages.description']) + expect(mockAudit).toHaveBeenNthCalledWith(2, qx, 'packagist', PURL, ['versions.number']) expect(mockMarkMetadata).toHaveBeenCalledWith( qx, PURL, @@ -227,24 +225,32 @@ describe('ingestOnePackagistMetadata', () => { ) }) - it('throws on a transient p2 result without marking scanned', async () => { + it('throws on a transient p2 result without marking scanned, but still audits phase 1', async () => { mockFetchStats.mockResolvedValue(statsJson as never) - mockPersistInfo.mockResolvedValue({ found: true, changedFields: [] }) + mockPersistInfo.mockResolvedValue({ found: true, changedFields: ['packages.description'] }) mockFetchP2.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 502' } as never) await expect(ingestOnePackagistMetadata(qx, candidate)).rejects.toThrow() expect(mockMarkMetadata).not.toHaveBeenCalled() + // phase-1 writes are already committed when the throw happens — a retry re-runs + // phase 1 idempotently and reports no changes, so this audit event can't be deferred + expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, ['packages.description']) }) }) // The monthly downloads-30d lane: dynamic fetch, window row only. describe('ingestOnePackagist30dWindow', () => { - it('persists the observed rolling window and marks the run processed', async () => { + it('persists the observed rolling window, audits the change, and marks the run processed', async () => { mockFetchStats.mockResolvedValue(statsJson as never) + mockPersist30d.mockResolvedValue(['downloads_last_30d.count', 'packages.downloads_last_30d']) await ingestOnePackagist30dWindow(qx, PURL, RUN_DATE) expect(mockPersist30d).toHaveBeenCalledWith(qx, PURL, 300, RUN_DATE) + expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, [ + 'downloads_last_30d.count', + 'packages.downloads_last_30d', + ]) expect(mockMark30d).toHaveBeenCalledWith( qx, PURL, @@ -284,12 +290,17 @@ describe('ingestOnePackagist30dWindow', () => { describe('ingestOnePackagistDailyDownload', () => { const candidate = { purl: PURL, packageId: '7' } - it('inserts the daily row and marks the run processed', async () => { + it('inserts the daily row, audits the change, and marks the run processed', async () => { mockFetchStats.mockResolvedValue(statsJson as never) + mockDaily.mockResolvedValue(['downloads_daily.date', 'downloads_daily.count']) await ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE) expect(mockDaily).toHaveBeenCalledWith(qx, '7', [{ day: RUN_DATE, downloads: 10 }]) + expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, [ + 'downloads_daily.date', + 'downloads_daily.count', + ]) expect(mockMarkDaily).toHaveBeenCalledWith( qx, PURL, @@ -297,12 +308,13 @@ describe('ingestOnePackagistDailyDownload', () => { ) }) - it('marks success without inserting when the registry reports no daily count', async () => { + it('marks success without inserting or auditing when the registry reports no daily count', async () => { mockFetchStats.mockResolvedValue({ package: { name: 'monolog/monolog' } } as never) await ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE) expect(mockDaily).not.toHaveBeenCalled() + expect(mockAudit).not.toHaveBeenCalled() expect(mockMarkDaily).toHaveBeenCalledWith( qx, PURL, diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts index 8ada433886..89db45b5f2 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts @@ -15,7 +15,7 @@ vi.mock('@crowd/data-access-layer/src/packages', () => ({ updatePackagistVersionAggregates: vi.fn(), upsertPackagistVersions: vi.fn(), getPackagistPackageIdsByNames: vi.fn(), - upsertVersionDependencies: vi.fn().mockResolvedValue(0), + upsertVersionDependencies: vi.fn().mockResolvedValue([]), })) const mockAggregates = vi.mocked(updatePackagistVersionAggregates) @@ -48,7 +48,7 @@ const expanded: PackagistExpandedVersion[] = [ beforeEach(() => { vi.clearAllMocks() - mockDeps.mockResolvedValue(0) + mockDeps.mockResolvedValue([]) }) // C2 + C3 — persistence wiring: aggregates on packages, version rows, dependency edges @@ -69,6 +69,7 @@ describe('persistPackagistMetadata', () => { ['phpunit/phpunit', '45'], ]), ) + mockDeps.mockResolvedValue(['package_dependencies.depends_on_id']) const result = await persistPackagistMetadata(qx, PURL, expanded) @@ -113,6 +114,8 @@ describe('persistPackagistMetadata', () => { expect(result.found).toBe(true) expect(result.unresolvedDependencyTargets).toBe(0) + // dependency-edge changes must reach the audit log alongside aggregates/versions + expect(result.changedFields).toContain('package_dependencies.depends_on_id') }) it('skips and counts dependency targets that do not resolve to a packages row', async () => { diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts index e2d87122b231649c422164f5f72eb4466a6d472b..d5248c81109b40e149976af105eac9041a99bc78 100644 GIT binary patch delta 210 zcmZ3d)vL2%3cG^>5ai}3XZt2*=9K_(YEiKsh#!<%oS##cni7_nQ<|y~t)7b{pORTz zkds&$pO=`MI{AV?_~z&ASC|476!i5K5XuyiGZORCQ;QXHON&bsic%AkGZacPQWX+Q zQ!+~wa`Mv^O7in{fEr4YGcxmlJfK`&L9Tv5WkIGwL1IZp@#H`*;mLY}f{M=hV5e!| Lbj0RuToqgZB|%M< delta 21 dcmeCxS*NvO3j5^Eg29`$Ij%5mp2c0v1pr@M2p9kW diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index 1331c6f2cf..808b81f91a 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -138,7 +138,8 @@ export async function ingestOnePackagistMetadata( const stats = normalizePackagistStats(info.value.package) const persistedInfo = await persistPackagistPackageInfo(qx, candidate.purl, stats) - const changedFields = [...persistedInfo.changedFields] + // Audit phase 1 immediately: those writes are already committed + await logAuditFieldChanges(qx, WORKER, candidate.purl, persistedInfo.changedFields) // Phase 2: p2 endpoint const p2 = await fetchWithFastRetry( @@ -150,13 +151,12 @@ export async function ingestOnePackagistMetadata( { purl: candidate.purl, statusCode: p2.error.statusCode, kind: p2.error.kind }, 'packagist metadata 4xx/malformed after fast retries — marking scanned and skipping', ) - // Phase-1 writes are already committed — their audit rows must not be dropped. - await logAuditFieldChanges(qx, WORKER, candidate.purl, changedFields) await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(p2.error)) return } let lastModified: string | null = null + let phase2ChangedFields: string[] = [] if (!isP2NotModified(p2.value)) { const expanded = expandComposerMetadata(p2.value.minifiedVersions) const persistResult = await persistPackagistMetadata(qx, candidate.purl, expanded) @@ -166,11 +166,11 @@ export async function ingestOnePackagistMetadata( 'packagist dependency targets not found in packages — edges skipped', ) } - changedFields.push(...persistResult.changedFields) + phase2ChangedFields = persistResult.changedFields lastModified = p2.value.lastModified } - await logAuditFieldChanges(qx, WORKER, candidate.purl, changedFields) + await logAuditFieldChanges(qx, WORKER, candidate.purl, phase2ChangedFields) if (lastModified) { await markPackagistMetadataScanned( qx, @@ -207,7 +207,13 @@ export async function ingestOnePackagist30dWindow( return } - await persistPackagist30dWindow(qx, purl, info.value.package.downloads?.monthly ?? null, runDate) + const changedFields = await persistPackagist30dWindow( + qx, + purl, + info.value.package.downloads?.monthly ?? null, + runDate, + ) + await logAuditFieldChanges(qx, WORKER, purl, changedFields) await markPackagist30dProcessed(qx, purl, { status: 'success', attempts: info.attempts }) } @@ -234,7 +240,10 @@ export async function ingestOnePackagistDailyDownload( const daily = info.value.package.downloads?.daily if (typeof daily === 'number') { - await insertDailyDownloads(qx, candidate.packageId, [{ day: runDate, downloads: daily }]) + const changedFields = await insertDailyDownloads(qx, candidate.packageId, [ + { day: runDate, downloads: daily }, + ]) + await logAuditFieldChanges(qx, WORKER, candidate.purl, changedFields) } await markPackagistDailyProcessed(qx, candidate.purl, { status: 'success', diff --git a/services/apps/packages_worker/src/packagist/downloads.ts b/services/apps/packages_worker/src/packagist/downloads.ts index e868c7b56e..5a4af0b820 100644 --- a/services/apps/packages_worker/src/packagist/downloads.ts +++ b/services/apps/packages_worker/src/packagist/downloads.ts @@ -21,18 +21,20 @@ export function monthlyWindowFor(runDate: string): { startDate: string; endDate: // One window row per purl per month, mirrored to packages.downloads_last_30d // (npm parity). A window already recorded for the month is never overwritten — -// the value is the observation closest to the boundary. +// the value is the observation closest to the boundary. Returns the changed +// fields (like the sibling persist*/upsert* functions) — the caller audits them. export async function persistPackagist30dWindow( qx: QueryExecutor, purl: string, monthly: number | null, runDate: string, -): Promise { - if (monthly == null) return +): Promise { + if (monthly == null) return [] const { startDate, endDate } = monthlyWindowFor(runDate) const existing = await getExistingLast30dEndDates(qx, purl, endDate, endDate) if (existing.length === 0) { - await upsertLast30dDownload(qx, purl, startDate, endDate, monthly, true) + return upsertLast30dDownload(qx, purl, startDate, endDate, monthly, true) } + return [] } diff --git a/services/apps/packages_worker/src/packagist/upsertMetadata.ts b/services/apps/packages_worker/src/packagist/upsertMetadata.ts index d14e25d2eb..625b5fa8b4 100644 --- a/services/apps/packages_worker/src/packagist/upsertMetadata.ts +++ b/services/apps/packages_worker/src/packagist/upsertMetadata.ts @@ -88,7 +88,8 @@ export async function persistPackagistMetadata( } if (edges.length > 0) { - await upsertVersionDependencies(qx, edges) + const depChanges = await upsertVersionDependencies(qx, edges) + changedFields.push(...depChanges) } } diff --git a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts index 2e3c14d5f5..6d62d9d269 100644 --- a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts +++ b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts @@ -53,7 +53,8 @@ export async function persistPackagistPackageInfo( // Step 3: Maintainers only for critical packages if (isCritical && stats.maintainers.length > 0) { - await upsertPackageMaintainers(qx, id, stats.maintainers, 'packagist') + const maintainerChanges = await upsertPackageMaintainers(qx, id, stats.maintainers, 'packagist') + changedFields.push(...maintainerChanges) } return { found: true, changedFields } diff --git a/services/libs/data-access-layer/src/packages/dependencies.ts b/services/libs/data-access-layer/src/packages/dependencies.ts index 38484bc326..adad1063fa 100644 --- a/services/libs/data-access-layer/src/packages/dependencies.ts +++ b/services/libs/data-access-layer/src/packages/dependencies.ts @@ -8,11 +8,15 @@ export interface VersionDependencyEdge { kind: 'direct' | 'dev' } +// Returns a batch-level changed-fields summary (like the sibling upsert* functions), +// not a per-edge diff: a batch can touch many edges across many versions in one call, +// so this reports whether ANY edge in the batch was newly inserted / had its +// constraint change, rather than one row per edge. export async function upsertVersionDependencies( qx: QueryExecutor, edges: VersionDependencyEdge[], -): Promise { - if (edges.length === 0) return 0 +): Promise { + if (edges.length === 0) return [] const packageIds = edges.map((e) => e.packageId) const versionIds = edges.map((e) => e.versionId) @@ -20,16 +24,35 @@ export async function upsertVersionDependencies( const constraints = edges.map((e) => e.constraint) const kinds = edges.map((e) => e.kind) - const result = await qx.result( - `INSERT INTO package_dependencies (package_id, version_id, depends_on_id, version_constraint, dependency_kind, is_optional, created_at, updated_at) - SELECT e.package_id, e.version_id, e.depends_on_id, e.version_constraint, e.dependency_kind, FALSE, NOW(), NOW() - FROM unnest($(packageIds)::bigint[], $(versionIds)::bigint[], $(dependsOnIds)::bigint[], - $(constraints)::text[], $(kinds)::text[]) - AS e(package_id, version_id, depends_on_id, version_constraint, dependency_kind) - ON CONFLICT (version_id, depends_on_id, dependency_kind) DO UPDATE SET - version_constraint = EXCLUDED.version_constraint, - updated_at = NOW()`, + const row: { changed_fields: string[] } = await qx.selectOne( + `WITH old AS ( + SELECT version_id, depends_on_id, dependency_kind, version_constraint + FROM package_dependencies + WHERE version_id = ANY($(versionIds)::bigint[]) + AND depends_on_id = ANY($(dependsOnIds)::bigint[]) + ), + ins AS ( + INSERT INTO package_dependencies (package_id, version_id, depends_on_id, version_constraint, dependency_kind, is_optional, created_at, updated_at) + SELECT e.package_id, e.version_id, e.depends_on_id, e.version_constraint, e.dependency_kind, FALSE, NOW(), NOW() + FROM unnest($(packageIds)::bigint[], $(versionIds)::bigint[], $(dependsOnIds)::bigint[], + $(constraints)::text[], $(kinds)::text[]) + AS e(package_id, version_id, depends_on_id, version_constraint, dependency_kind) + ON CONFLICT (version_id, depends_on_id, dependency_kind) DO UPDATE SET + version_constraint = EXCLUDED.version_constraint, + updated_at = NOW() + RETURNING version_id, depends_on_id, dependency_kind, version_constraint + ) + SELECT array_remove(ARRAY[ + CASE WHEN bool_or(o.version_id IS NULL) THEN 'package_dependencies.depends_on_id' END, + CASE WHEN bool_or(o.version_id IS NOT NULL AND o.version_constraint IS DISTINCT FROM ins.version_constraint) + THEN 'package_dependencies.version_constraint' END + ], NULL) AS changed_fields + FROM ins + LEFT JOIN old o + ON o.version_id = ins.version_id + AND o.depends_on_id = ins.depends_on_id + AND o.dependency_kind = ins.dependency_kind`, { packageIds, versionIds, dependsOnIds, constraints, kinds }, ) - return result + return row.changed_fields } From 44dad1b79c4c148adbec05891572365c5a96b3d5 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 13:49:43 +0200 Subject: [PATCH 20/56] fix: enforce SCM host allowlist, clear stale repo links Signed-off-by: anilb --- .../__tests__/persistPackageInfo.test.ts | Bin 5645 -> 6958 bytes .../src/packagist/upsertPackageInfo.ts | 21 ++++++++++++++---- .../data-access-layer/src/packages/repos.ts | 15 +++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts index d5248c81109b40e149976af105eac9041a99bc78..62de1d7b81567dc7ec3a2d6eec57fdf8d0176c8d 100644 GIT binary patch delta 738 zcmbV}&x#X45XRXAb%`dsf!#I!gVM_+kjxwe0|5n5FXGA)T|6i*y)(5*nNIiE-7~XM z7JUIhX%C(RUqF<+g%^DU@#-@Ob|-rQ|>VkxwNv5ZIlV}GUq*CD6SzhTl`ZYpn8t}>B8%#kS# zOiUnEn3K#jpLCMr;UEP4ohzf!1r)p6L7Yn28L;bP+vUb5HwT6vWFO{Dk zw(XPE^9|=Wm70uHlU>DoATrM3sMWGR*Y@qlTK(kQt^IN4V!0c%?9S=VRyv2?o3(~M z?@~0^nP!kk)F7k*qcXQI1~C&-P|P^J zzyyZUJ<7~zO3)L-gIn9cIF|({S7{bBqKLH4uw#_7h$>Pza3wW?-Tpmo-)-Erf2!NI zwR!ec6fLOxyEuNg4J~=@Y?vXrB%KL`jKfRF6mQQg^dOLa-vJy!%1FVl*Y6#G4<||{ zm@_a}R2t!c5ziF1;hAUhbPRHQsOW$klSlAe;jzNfHG-irsqRHlQ54}A6Jj1Rlxh&o eXzv^v6q=;y-Oom7`DXWj4qNI5(;MsGPy7XBpaF*f delta 86 zcmZ2y)~hoiX=A-D)8rpa4>yalaB@kMXQbvSlw_nT6eK2RC#I(=q@*V2Bo?Jk4&*DJ q>>zA3*^}RBavU$` { + const rowCount = await qx.result( + `DELETE FROM package_repos WHERE package_id = $(packageId)::bigint AND source = 'declared'`, + { packageId }, + ) + return rowCount > 0 ? ['package_repos.repo_id'] : [] +} + export async function upsertPackageRepo( qx: QueryExecutor, packageId: string, From 34056416873d34282d340bd66d7f8e230c7ddc83 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 13:49:44 +0200 Subject: [PATCH 21/56] fix: reconcile removed packagist dependency edges Signed-off-by: anilb --- .../__tests__/persistMetadata.test.ts | 31 ++++++++++-- .../src/packagist/upsertMetadata.ts | 47 +++++++++++-------- .../src/packages/dependencies.ts | 35 ++++++++++++++ 3 files changed, 88 insertions(+), 25 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts index 89db45b5f2..8d89e048a2 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts @@ -2,9 +2,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { getPackagistPackageIdsByNames, + reconcileVersionDependencies, updatePackagistVersionAggregates, upsertPackagistVersions, - upsertVersionDependencies, } from '@crowd/data-access-layer/src/packages' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' @@ -15,13 +15,13 @@ vi.mock('@crowd/data-access-layer/src/packages', () => ({ updatePackagistVersionAggregates: vi.fn(), upsertPackagistVersions: vi.fn(), getPackagistPackageIdsByNames: vi.fn(), - upsertVersionDependencies: vi.fn().mockResolvedValue([]), + reconcileVersionDependencies: vi.fn().mockResolvedValue([]), })) const mockAggregates = vi.mocked(updatePackagistVersionAggregates) const mockVersions = vi.mocked(upsertPackagistVersions) const mockIds = vi.mocked(getPackagistPackageIdsByNames) -const mockDeps = vi.mocked(upsertVersionDependencies) +const mockDeps = vi.mocked(reconcileVersionDependencies) const qx = {} as QueryExecutor const PURL = 'pkg:composer/monolog/monolog' @@ -102,7 +102,9 @@ describe('persistPackagistMetadata', () => { expect([...requestedNames].sort()).toEqual(['phpunit/phpunit', 'psr/log']) expect(mockDeps).toHaveBeenCalledTimes(1) - const edges = mockDeps.mock.calls[0][1] + // reconciliation is scoped to exactly the versions refreshed this pass + expect(mockDeps.mock.calls[0][1]).toEqual(expect.arrayContaining(['21', '20'])) + const edges = mockDeps.mock.calls[0][2] expect(edges).toEqual( expect.arrayContaining([ { packageId: '9', versionId: '21', dependsOnId: '44', constraint: '^3.0', kind: 'direct' }, @@ -132,12 +134,31 @@ describe('persistPackagistMetadata', () => { const result = await persistPackagistMetadata(qx, PURL, expanded) - const edges = mockDeps.mock.calls[0][1] + const edges = mockDeps.mock.calls[0][2] expect(edges).toHaveLength(2) expect(edges.every((e) => e.dependsOnId === '44')).toBe(true) expect(result.unresolvedDependencyTargets).toBe(1) }) + it('reconciles (deletes stale edges) even when a version now declares zero dependencies', async () => { + mockAggregates.mockResolvedValue({ id: '9', changedFields: [] }) + mockVersions.mockResolvedValue({ + changedFields: [], + versionIds: [{ number: '1.0.0', id: '20' }], + }) + mockDeps.mockResolvedValue(['package_dependencies.depends_on_id']) + + // no require/require-dev at all — a prior run's edges for this version must still + // be reconciled away, not left stale just because nothing new was declared + const result = await persistPackagistMetadata(qx, PURL, [ + { version: '1.0.0', version_normalized: '1.0.0.0' }, + ]) + + expect(mockIds).not.toHaveBeenCalled() + expect(mockDeps).toHaveBeenCalledWith(qx, ['20'], []) + expect(result.changedFields).toContain('package_dependencies.depends_on_id') + }) + it('returns found=false and writes nothing when the packages row is missing', async () => { mockAggregates.mockResolvedValue(null) diff --git a/services/apps/packages_worker/src/packagist/upsertMetadata.ts b/services/apps/packages_worker/src/packagist/upsertMetadata.ts index 625b5fa8b4..d2dfab9286 100644 --- a/services/apps/packages_worker/src/packagist/upsertMetadata.ts +++ b/services/apps/packages_worker/src/packagist/upsertMetadata.ts @@ -1,8 +1,8 @@ import { getPackagistPackageIdsByNames, + reconcileVersionDependencies, updatePackagistVersionAggregates, upsertPackagistVersions, - upsertVersionDependencies, } from '@crowd/data-access-layer/src/packages' import type { VersionDependencyEdge } from '@crowd/data-access-layer/src/packages' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' @@ -67,30 +67,37 @@ export async function persistPackagistMetadata( } } + // Reconciled for every version being refreshed (not just ones with a resolved + // dependency this pass) — a version whose manifest drops a requirement, or all of + // them, must have its stale package_dependencies rows removed, not just the newly + // declared ones upserted. let unresolvedDependencyTargets = 0 - if (pending.length > 0) { - const idMap = await getPackagistPackageIdsByNames(qx, Array.from(targetNames)) - + if (versionIds.length > 0) { const edges: VersionDependencyEdge[] = [] - for (const { versionId, dep } of pending) { - const dependsOnId = idMap.get(dep.name) - if (!dependsOnId) { - unresolvedDependencyTargets++ - continue + if (pending.length > 0) { + const idMap = await getPackagistPackageIdsByNames(qx, Array.from(targetNames)) + for (const { versionId, dep } of pending) { + const dependsOnId = idMap.get(dep.name) + if (!dependsOnId) { + unresolvedDependencyTargets++ + continue + } + edges.push({ + packageId: agg.id, + versionId, + dependsOnId, + constraint: dep.constraint, + kind: dep.kind, + }) } - edges.push({ - packageId: agg.id, - versionId, - dependsOnId, - constraint: dep.constraint, - kind: dep.kind, - }) } - if (edges.length > 0) { - const depChanges = await upsertVersionDependencies(qx, edges) - changedFields.push(...depChanges) - } + const depChanges = await reconcileVersionDependencies( + qx, + versionIds.map((v) => v.id), + edges, + ) + changedFields.push(...depChanges) } return { found: true, changedFields, unresolvedDependencyTargets } diff --git a/services/libs/data-access-layer/src/packages/dependencies.ts b/services/libs/data-access-layer/src/packages/dependencies.ts index adad1063fa..298ed1f043 100644 --- a/services/libs/data-access-layer/src/packages/dependencies.ts +++ b/services/libs/data-access-layer/src/packages/dependencies.ts @@ -56,3 +56,38 @@ export async function upsertVersionDependencies( ) return row.changed_fields } + +// Reconciles ALL dependency edges for a set of versions being refreshed: deletes any +// existing package_dependencies row for those versions that isn't in `edges` (a +// requirement removed from the manifest, or a version that now declares none at all), +// then upserts the current edges. Scoped to `versionIds` — version_id has a supporting +// btree index, so this stays a bounded, per-package operation despite the +// depends_on_id HASH-partitioning. +export async function reconcileVersionDependencies( + qx: QueryExecutor, + versionIds: string[], + edges: VersionDependencyEdge[], +): Promise { + if (versionIds.length === 0) return [] + + const deletedCount = await qx.result( + `DELETE FROM package_dependencies + WHERE version_id = ANY($(versionIds)::bigint[]) + AND NOT EXISTS ( + SELECT 1 FROM unnest($(keepVersionIds)::bigint[], $(keepDependsOnIds)::bigint[], $(keepKinds)::text[]) + AS keep(version_id, depends_on_id, dependency_kind) + WHERE keep.version_id = package_dependencies.version_id + AND keep.depends_on_id = package_dependencies.depends_on_id + AND keep.dependency_kind = package_dependencies.dependency_kind + )`, + { + versionIds, + keepVersionIds: edges.map((e) => e.versionId), + keepDependsOnIds: edges.map((e) => e.dependsOnId), + keepKinds: edges.map((e) => e.kind), + }, + ) + + const upsertChanges = await upsertVersionDependencies(qx, edges) + return deletedCount > 0 ? [...upsertChanges, 'package_dependencies.depends_on_id'] : upsertChanges +} From 544cbe60310bf0998536319dc8256c383c11be49 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 13:49:53 +0200 Subject: [PATCH 22/56] fix: per-item retries and download-lane keyset cursors Signed-off-by: anilb --- .../packagist/__tests__/dueSelection.test.ts | 16 +++++---- .../src/packagist/__tests__/ingest.test.ts | 26 +++++++++++++- .../src/packagist/activities.ts | 35 +++++++++++++++---- .../src/packagist/workflows.ts | 23 +++++++++--- .../src/packages/packagistPackageState.ts | 12 ++++--- 5 files changed, 89 insertions(+), 23 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts index a9119a078f..31d6ada7a9 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts @@ -102,17 +102,19 @@ describe('markPackagistMetadataScanned', () => { // Monthly downloads-30d lane: npm-style breadth — every package is due once per run // (watermark older than the run's cutoff, or never run). describe('getPackagist30dDuePurls / markPackagist30dProcessed', () => { - it('selects packagist purls whose 30d watermark is older than the cutoff', async () => { + it('selects packagist purls whose 30d watermark is older than the cutoff, keyset-paginated by purl', async () => { qx.select.mockResolvedValue([{ purl: 'pkg:composer/a/x' }]) - const purls = await getPackagist30dDuePurls(asQx(qx), '2026-07-01T03:53:00Z', 50) + const purls = await getPackagist30dDuePurls(asQx(qx), '2026-07-01T03:53:00Z', '', 50) expect(purls).toEqual(['pkg:composer/a/x']) const [sql, params] = qx.select.mock.calls[0] expect(sql).toMatch(/ecosystem = 'packagist'/) expect(sql).toMatch(/downloads_30d_last_run_at/) + expect(sql).toMatch(/p\.purl > \$\(afterPurl\)/) + expect(sql).toMatch(/ORDER BY\s+p\.purl/i) expect(sql).toMatch(/LIMIT/i) - expect(params).toMatchObject({ cutoff: '2026-07-01T03:53:00Z', batchSize: 50 }) + expect(params).toMatchObject({ cutoff: '2026-07-01T03:53:00Z', afterPurl: '', batchSize: 50 }) }) it('bumps the 30d watermark with the run result', async () => { @@ -136,17 +138,19 @@ describe('getPackagist30dDuePurls / markPackagist30dProcessed', () => { // Daily downloads lane: critical slice only, cutoff watermark, returns the package id // the downloads_daily insert needs. describe('getPackagistDailyDownloadsDue / markPackagistDailyProcessed', () => { - it('selects critical packagist packages due for a daily capture, with their ids', async () => { + it('selects critical packagist packages due for a daily capture, keyset-paginated by purl', async () => { qx.select.mockResolvedValue([{ purl: 'pkg:composer/a/x', package_id: '7' }]) - const due = await getPackagistDailyDownloadsDue(asQx(qx), '2026-07-15T06:23:00Z', 50) + const due = await getPackagistDailyDownloadsDue(asQx(qx), '2026-07-15T06:23:00Z', '', 50) expect(due).toEqual([{ purl: 'pkg:composer/a/x', packageId: '7' }]) const [sql, params] = qx.select.mock.calls[0] expect(sql).toMatch(/ecosystem = 'packagist'/) expect(sql).toMatch(/is_critical/) expect(sql).toMatch(/daily_downloads_last_run_at/) - expect(params).toMatchObject({ cutoff: '2026-07-15T06:23:00Z', batchSize: 50 }) + expect(sql).toMatch(/p\.purl > \$\(afterPurl\)/) + expect(sql).toMatch(/ORDER BY\s+p\.purl/i) + expect(params).toMatchObject({ cutoff: '2026-07-15T06:23:00Z', afterPurl: '', batchSize: 50 }) }) it('bumps the daily watermark with the run result', async () => { diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts index 3dcd382e72..2fb8d36c23 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -343,7 +343,31 @@ describe('ingestPackagistItemsConcurrently', () => { const ingest = makeIngest() const onGiveUp = vi.fn().mockResolvedValue(undefined) - await expect(ingestPackagistItemsConcurrently(items, 1, 2, ingest, onGiveUp)).rejects.toThrow() + await expect(ingestPackagistItemsConcurrently(items, 1, 2, ingest, onGiveUp)).rejects.toThrow( + 'transient', + ) + expect(onGiveUp).not.toHaveBeenCalled() + // every item — including ones scheduled after 'bad' — must get a genuine attempt + // this round; `attempt` tracks the whole batch, not each item, so a rethrow from + // inside the concurrency pool must never starve later items of their own try. + expect(ingest).toHaveBeenCalledTimes(6) + }) + + it('still attempts every later item when the FIRST item in the batch fails (sequential, deterministic)', async () => { + const ingest = vi.fn((item: string) => + item === 'bad' ? Promise.reject(new Error('transient')) : Promise.resolve(), + ) + const onGiveUp = vi.fn().mockResolvedValue(undefined) + + // concurrency=1 forces full sequencing: mapWithConcurrency awaits each item's + // settlement (including the old code's rethrow-triggered `failed=true`) before + // ever considering the next index, so this deterministically proves items after + // the failing one still get scheduled instead of being starved. + await expect( + ingestPackagistItemsConcurrently(['bad', 'ok1', 'ok2'], 1, 1, ingest, onGiveUp), + ).rejects.toThrow('transient') + + expect(ingest).toHaveBeenCalledTimes(3) expect(onGiveUp).not.toHaveBeenCalled() }) diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index 808b81f91a..e43ce35bdb 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -258,12 +258,22 @@ export async function ingestPackagistItemsConcurrently( ingest: (item: T) => Promise, onGiveUp: (item: T, err: unknown) => Promise, ): Promise { + // mapWithConcurrency stops scheduling new items after the first rejection from the + // wrapped callback. `attempt` tracks the whole batch activity, not each item, so a + // rethrow here on an early item would starve every later item in the array of a + // genuine try this round. Never rethrow from inside the callback — collect the first + // retryable failure and decide once, after every item has actually been attempted. + let firstRetryableError: unknown + await mapWithConcurrency(items, concurrency, async (item) => { try { await ingest(item) } catch (err) { // Retry via Temporal while attempts remain; then give up and continue - if (attempt < INGEST_MAX_ATTEMPTS) throw err + if (attempt < INGEST_MAX_ATTEMPTS) { + if (firstRetryableError === undefined) firstRetryableError = err + return + } log.warn( { item: String(item), attempt, err: String(err) }, 'packagist item failed after max attempts — giving up', @@ -271,6 +281,8 @@ export async function ingestPackagistItemsConcurrently( await onGiveUp(item, err) } }) + + if (firstRetryableError !== undefined) throw firstRetryableError } export async function runPackagistPackageSeed(): Promise<{ discovered: number; invalid: number }> { @@ -336,15 +348,19 @@ export async function ingestPackagistMetadataBatch( log.info({ count: candidates.length }, 'Ingested Packagist metadata batch') } -export async function getPackagist30dBatch(cutoff: string, batchSize: number): Promise { +export async function getPackagist30dBatch( + cutoff: string, + afterPurl: string, + batchSize: number, +): Promise<{ purls: string[]; nextCursor: string }> { const qx = await getPackagesDb() - return getPackagist30dDuePurls(qx, cutoff, batchSize) + const purls = await getPackagist30dDuePurls(qx, cutoff, afterPurl, batchSize) + return { purls, nextCursor: purls.length ? purls[purls.length - 1] : afterPurl } } -export async function ingestPackagist30dBatch(purls: string[]): Promise { +export async function ingestPackagist30dBatch(purls: string[], runDate: string): Promise { if (purls.length === 0) return const qx = await getPackagesDb() - const runDate = new Date().toISOString().slice(0, 10) const attempt = Context.current().info.attempt await ingestPackagistItemsConcurrently( @@ -365,10 +381,15 @@ export async function ingestPackagist30dBatch(purls: string[]): Promise { export async function getPackagistDailyBatch( cutoff: string, + afterPurl: string, batchSize: number, -): Promise { +): Promise<{ candidates: PackagistDailyCandidate[]; nextCursor: string }> { const qx = await getPackagesDb() - return getPackagistDailyDownloadsDue(qx, cutoff, batchSize) + const candidates = await getPackagistDailyDownloadsDue(qx, cutoff, afterPurl, batchSize) + return { + candidates, + nextCursor: candidates.length ? candidates[candidates.length - 1].purl : afterPurl, + } } export async function ingestPackagistDailyBatch( diff --git a/services/apps/packages_worker/src/packagist/workflows.ts b/services/apps/packages_worker/src/packagist/workflows.ts index c84fa215bb..00b37bcca7 100644 --- a/services/apps/packages_worker/src/packagist/workflows.ts +++ b/services/apps/packages_worker/src/packagist/workflows.ts @@ -28,6 +28,7 @@ interface MetadataState { interface DownloadsState { cutoff?: string + cursor?: string } export async function seedPackagistPackages(): Promise { @@ -79,17 +80,23 @@ export async function ingestPackagistMetadata(state: MetadataState = {}): Promis // due-selection drains the whole universe exactly once per cron fire. export async function ingestPackagistDownloads30d(state: DownloadsState = {}): Promise { const cutoff = state.cutoff ?? (await acts.packagistCurrentTimestamp()) + // Packagist's monthly window is labeled by calendar month (see downloads.ts), so the + // write-date must come from the run's fixed cutoff — a drain that runs past real UTC + // midnight on the 1st must not let later batches slide into the next month's window. + const runDate = cutoff.slice(0, 10) + let cursor = state.cursor || '' const stopAfterFirstPage = await acts.packagistStopAfterFirstPage() for (let r = 0; r < ROUNDS_PER_RUN; r++) { - const purls = await acts.getPackagist30dBatch(cutoff, INGEST_BATCH) + const { purls, nextCursor } = await acts.getPackagist30dBatch(cutoff, cursor, INGEST_BATCH) if (purls.length === 0) return - await acts.ingestPackagist30dBatch(purls) + await acts.ingestPackagist30dBatch(purls, runDate) + cursor = nextCursor if (stopAfterFirstPage) return if (purls.length < INGEST_BATCH) return } - await continueAsNew({ cutoff }) + await continueAsNew({ cutoff, cursor }) } // Daily downloads capture for the critical slice. @@ -101,15 +108,21 @@ export async function ingestPackagistDownloadsDaily(state: DownloadsState = {}): // derive the write-date from the run's fixed cutoff instead of re-reading the clock // per batch, so a drain that runs past UTC midnight still tags every row consistently. const runDate = cutoff.slice(0, 10) + let cursor = state.cursor || '' const stopAfterFirstPage = await acts.packagistStopAfterFirstPage() for (let r = 0; r < ROUNDS_PER_RUN; r++) { - const candidates = await acts.getPackagistDailyBatch(cutoff, INGEST_BATCH) + const { candidates, nextCursor } = await acts.getPackagistDailyBatch( + cutoff, + cursor, + INGEST_BATCH, + ) if (candidates.length === 0) return await acts.ingestPackagistDailyBatch(candidates, runDate) + cursor = nextCursor if (stopAfterFirstPage) return if (candidates.length < INGEST_BATCH) return } - await continueAsNew({ cutoff }) + await continueAsNew({ cutoff, cursor }) } diff --git a/services/libs/data-access-layer/src/packages/packagistPackageState.ts b/services/libs/data-access-layer/src/packages/packagistPackageState.ts index 2765159e02..96991f0ef8 100644 --- a/services/libs/data-access-layer/src/packages/packagistPackageState.ts +++ b/services/libs/data-access-layer/src/packages/packagistPackageState.ts @@ -69,6 +69,7 @@ export async function getPackagistMetadataDuePurls( export async function getPackagist30dDuePurls( qx: QueryExecutor, cutoff: string, + afterPurl: string, batchSize: number, ): Promise { const rows: Array<{ purl: string }> = await qx.select( @@ -76,13 +77,14 @@ export async function getPackagist30dDuePurls( FROM packages p LEFT JOIN packagist_package_state s ON s.purl = p.purl WHERE p.ecosystem = 'packagist' + AND p.purl > $(afterPurl) AND ( s.downloads_30d_last_run_at IS NULL OR s.downloads_30d_last_run_at < $(cutoff)::timestamptz ) - ORDER BY s.downloads_30d_last_run_at ASC NULLS FIRST, p.purl + ORDER BY p.purl LIMIT $(batchSize)`, - { cutoff, batchSize }, + { cutoff, afterPurl, batchSize }, ) return rows.map((r) => r.purl) } @@ -105,6 +107,7 @@ export async function markPackagist30dProcessed( export async function getPackagistDailyDownloadsDue( qx: QueryExecutor, cutoff: string, + afterPurl: string, batchSize: number, ): Promise { const rows: Array<{ purl: string; package_id: string }> = await qx.select( @@ -113,13 +116,14 @@ export async function getPackagistDailyDownloadsDue( LEFT JOIN packagist_package_state s ON s.purl = p.purl WHERE p.ecosystem = 'packagist' AND p.is_critical = TRUE + AND p.purl > $(afterPurl) AND ( s.daily_downloads_last_run_at IS NULL OR s.daily_downloads_last_run_at < $(cutoff)::timestamptz ) - ORDER BY s.daily_downloads_last_run_at ASC NULLS FIRST, p.purl + ORDER BY p.purl LIMIT $(batchSize)`, - { cutoff, batchSize }, + { cutoff, afterPurl, batchSize }, ) return rows.map((r) => ({ purl: r.purl, packageId: r.package_id })) } From eb7d1257dbc2821d71886b405f1bf9dcce5e4192 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 14:33:04 +0200 Subject: [PATCH 23/56] refactor: remove dead purlFromPackagistName helper Signed-off-by: anilb --- .../packages_worker/src/packagist/__tests__/normalize.test.ts | 4 +--- services/apps/packages_worker/src/packagist/normalize.ts | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts index 02169f5cb6..0b17389691 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts @@ -7,13 +7,11 @@ import { isPackagistPrerelease, normalizePackagistStats, packagistNameFromPurl, - purlFromPackagistName, } from '../normalize' import type { PackagistPackageInfo } from '../types' describe('purl helpers', () => { - it('round-trips vendor/name through pkg:composer purls', () => { - expect(purlFromPackagistName('monolog/monolog')).toBe('pkg:composer/monolog/monolog') + it('strips the pkg:composer prefix to recover vendor/name', () => { expect(packagistNameFromPurl('pkg:composer/monolog/monolog')).toBe('monolog/monolog') expect(packagistNameFromPurl('pkg:composer/symfony/http-kernel')).toBe('symfony/http-kernel') }) diff --git a/services/apps/packages_worker/src/packagist/normalize.ts b/services/apps/packages_worker/src/packagist/normalize.ts index 642e9079f7..a21c73258a 100644 --- a/services/apps/packages_worker/src/packagist/normalize.ts +++ b/services/apps/packages_worker/src/packagist/normalize.ts @@ -12,10 +12,6 @@ export function packagistNameFromPurl(purl: string): string { return purl.slice(PURL_COMPOSER_PREFIX.length) } -export function purlFromPackagistName(name: string): string { - return PURL_COMPOSER_PREFIX + name -} - function blankToNull(s: string | null | undefined): string | null { if (s == null) return null const t = s.trim() From 7009d3d472d1fe7bbb6823ee2a5e55accf06669f Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 14:33:06 +0200 Subject: [PATCH 24/56] refactor: simplify packagist scan/give-up call sites Signed-off-by: anilb --- .../src/packagist/__tests__/ingest.test.ts | 2 ++ .../src/packagist/activities.ts | 33 ++++++++----------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts index 2fb8d36c23..b04bb6ace2 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -139,10 +139,12 @@ describe('ingestOnePackagistMetadata', () => { expect(mockPersistInfo).toHaveBeenCalled() expect(mockPersistMetadata).not.toHaveBeenCalled() + // no fresh Last-Modified to replay when p2 wasn't refetched expect(mockMarkMetadata).toHaveBeenCalledWith( qx, PURL, expect.objectContaining({ status: 'success' }), + null, ) }) diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index e43ce35bdb..e41451357e 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -104,10 +104,10 @@ async function fetchWithFastRetry( } } -function giveUpResult(error: FetchError): PackagistRunResult { +function giveUpResult(error: FetchError, attempts: number): PackagistRunResult { return { status: 'error', - attempts: INGEST_4XX_ATTEMPTS, + attempts, httpStatus: error.statusCode, errorKind: error.kind, message: error.message, @@ -132,7 +132,7 @@ export async function ingestOnePackagistMetadata( { purl: candidate.purl, statusCode: info.error.statusCode, kind: info.error.kind }, 'packagist package info 4xx/malformed after fast retries — marking scanned and skipping', ) - await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(info.error)) + await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(info.error, info.attempts)) return } @@ -151,7 +151,7 @@ export async function ingestOnePackagistMetadata( { purl: candidate.purl, statusCode: p2.error.statusCode, kind: p2.error.kind }, 'packagist metadata 4xx/malformed after fast retries — marking scanned and skipping', ) - await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(p2.error)) + await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(p2.error, p2.attempts)) return } @@ -171,19 +171,14 @@ export async function ingestOnePackagistMetadata( } await logAuditFieldChanges(qx, WORKER, candidate.purl, phase2ChangedFields) - if (lastModified) { - await markPackagistMetadataScanned( - qx, - candidate.purl, - { status: 'success', attempts: p2.attempts }, - lastModified, - ) - } else { - await markPackagistMetadataScanned(qx, candidate.purl, { - status: 'success', - attempts: p2.attempts, - }) - } + // markPackagistMetadataScanned treats a null 4th arg the same as omitting it + // (COALESCEd against the stored value), so lastModified can always be passed. + await markPackagistMetadataScanned( + qx, + candidate.purl, + { status: 'success', attempts: p2.attempts }, + lastModified, + ) } // The monthly downloads-30d lane: dynamic fetch, one window row per purl per month. @@ -203,7 +198,7 @@ export async function ingestOnePackagist30dWindow( { purl, statusCode: info.error.statusCode, kind: info.error.kind }, 'packagist 30d downloads 4xx/malformed after fast retries — marking processed and skipping', ) - await markPackagist30dProcessed(qx, purl, giveUpResult(info.error)) + await markPackagist30dProcessed(qx, purl, giveUpResult(info.error, info.attempts)) return } @@ -234,7 +229,7 @@ export async function ingestOnePackagistDailyDownload( { purl: candidate.purl, statusCode: info.error.statusCode, kind: info.error.kind }, 'packagist daily downloads 4xx/malformed after fast retries — marking processed and skipping', ) - await markPackagistDailyProcessed(qx, candidate.purl, giveUpResult(info.error)) + await markPackagistDailyProcessed(qx, candidate.purl, giveUpResult(info.error, info.attempts)) return } From 57c33ff1f0f79a2b9821e0ccdeec03f07f3e97cb Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 14:33:14 +0200 Subject: [PATCH 25/56] docs: trim overlong packagist comments Signed-off-by: anilb --- .../apps/packages_worker/src/packagist/schedule.ts | 7 ++----- .../apps/packages_worker/src/packagist/workflows.ts | 13 ++++--------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/schedule.ts b/services/apps/packages_worker/src/packagist/schedule.ts index e2556f0d66..e7a526db5a 100644 --- a/services/apps/packages_worker/src/packagist/schedule.ts +++ b/services/apps/packages_worker/src/packagist/schedule.ts @@ -7,11 +7,8 @@ import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/clien export const PACKAGIST_CRONS = { seed: '17 2 * * 0', downloads30d: '53 3 1 * *', - // Late in the UTC day on purpose: Packagist's `daily` figure is `today_so_far + - // yesterday_total * dayRatio` (DownloadManager.php), where dayRatio shrinks toward 0 - // as the day progresses — querying at 22:23 UTC (dayRatio ~0.08) captures a figure - // that's ~92% today's real data instead of ~74% borrowed from yesterday at the old - // 06:23 slot, while keeping a ~1h37m buffer before UTC midnight for the drain to finish. + // Late in the UTC day on purpose: Packagist's `daily` figure is mostly real data by + // 22:23 (vs. mostly borrowed from yesterday earlier on), with buffer before midnight. downloadsDaily: '23 22 * * *', } diff --git a/services/apps/packages_worker/src/packagist/workflows.ts b/services/apps/packages_worker/src/packagist/workflows.ts index 00b37bcca7..6365a241ab 100644 --- a/services/apps/packages_worker/src/packagist/workflows.ts +++ b/services/apps/packages_worker/src/packagist/workflows.ts @@ -34,15 +34,10 @@ interface DownloadsState { export async function seedPackagistPackages(): Promise { await acts.runPackagistPackageSeed() - // Chain the metadata drain off a completed seed instead of a second cron: newly - // discovered packages are rows before the p2 crawl resolves dependency targets. - // Started (not awaited) with ABANDON — the drain runs for hours and must outlive - // this workflow; the state watermarks make the pass self-advancing regardless. - // Fixed workflowId + ALLOW_DUPLICATE: starting a duplicate against a still-RUNNING - // execution with the same id always throws WorkflowExecutionAlreadyStartedError - // regardless of reuse policy, so a drain that outlasts the week makes next - // Sunday's seed skip its chain-start instead of doubling the crawl; ALLOW_DUPLICATE - // just lets the id be reused once that prior execution has actually closed. + // Chain the drain off seed completion (not a cron) so newly discovered packages exist + // as rows first. ABANDON so it outlives this workflow; fixed id + ALLOW_DUPLICATE means + // a drain that outlasts the week makes next Sunday's seed skip its start instead of + // doubling the crawl (a still-RUNNING id always throws regardless of reuse policy). try { await startChild(ingestPackagistMetadata, { workflowId: 'packagist-metadata-drain', From 5cd4ea12298823b80fbe9b9a0acb53d4d42d3a46 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 16:51:27 +0200 Subject: [PATCH 26/56] fix: bump last_synced_at on packagist version cleanup Signed-off-by: anilb --- services/libs/data-access-layer/src/packages/versions.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/libs/data-access-layer/src/packages/versions.ts b/services/libs/data-access-layer/src/packages/versions.ts index 2a127dabd7..523454da80 100644 --- a/services/libs/data-access-layer/src/packages/versions.ts +++ b/services/libs/data-access-layer/src/packages/versions.ts @@ -153,10 +153,11 @@ export async function upsertPackagistVersions( // Clear a stale is_latest on every OTHER version of this package. Anchored on the declared // latest (latestNumber) — NOT on what's in this batch — so a previously-latest version whose // records were updated differently can't keep is_latest = true alongside the new latest. When - // no latest is known, leave flags untouched rather than wipe all. + // no latest is known, leave flags untouched rather than wipe all. last_synced_at must move too + // — it's the Tinybird ENGINE_VER, and this row's is_latest is a real change. if (latestNumber != null) { await qx.result( - `UPDATE versions SET is_latest = false + `UPDATE versions SET is_latest = false, last_synced_at = NOW() WHERE package_id = $(packageId)::bigint AND is_latest = true AND number <> $(latestNumber)`, From e9cad6ddcb47b38c1715419137a6c669de1dcc01 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 16:51:28 +0200 Subject: [PATCH 27/56] fix: add retry policy to packagist schedules Signed-off-by: anilb --- services/apps/packages_worker/src/packagist/schedule.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/apps/packages_worker/src/packagist/schedule.ts b/services/apps/packages_worker/src/packagist/schedule.ts index e7a526db5a..f81ba421cf 100644 --- a/services/apps/packages_worker/src/packagist/schedule.ts +++ b/services/apps/packages_worker/src/packagist/schedule.ts @@ -60,6 +60,11 @@ export async function schedulePackagistIngest(): Promise { workflowType: schedule.workflowType, taskQueue: 'packagist-worker', workflowRunTimeout: '24 hours', + retry: { + initialInterval: '30 seconds', + backoffCoefficient: 2, + maximumAttempts: 3, + }, args: schedule.args, }, }) From ad5e92e33f8ffe350d82d8ddc40dd61f927ff448 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 16:51:30 +0200 Subject: [PATCH 28/56] docs: fix stale packagist 30d pinning note Signed-off-by: anilb --- services/apps/packages_worker/src/packagist/README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index 4e167f2e69..8c31c6a442 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -135,10 +135,9 @@ never overwritten — the first observation (closest to the boundary) wins. Packagist keeps no history, so unlike npm there is no backfill lane; a missed month stays missed. State: `downloads_30d_last_run_at` + `downloads_30d_run_result`. -Unlike the daily lane (see below), the write-date here isn't pinned to a fixed -run cutoff — if the full-catalog drain ran long enough, the labeled window -could drift forward from what was true when the run started. Low risk given -the run should finish in hours, not weeks, but worth knowing. +Like the daily lane (see below), the write-date is pinned to the run's fixed +cutoff and threaded through every batch, so the labeled window stays correct +even if the drain runs long. --- From 3a9d2bc8013091b61cc35a1b75a106a1de6ef303 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 16:55:35 +0200 Subject: [PATCH 29/56] fix: don't defer packagist refresh on p2-only failure Signed-off-by: anilb --- .../src/packagist/__tests__/ingest.test.ts | 6 ++++++ .../packages_worker/src/packagist/activities.ts | 11 ++++++++++- .../src/packages/packagistPackageState.ts | 16 +++++++++++++--- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts index b04bb6ace2..b76ce330b5 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -197,10 +197,14 @@ describe('ingestOnePackagistMetadata', () => { await p expect(mockPersistMetadata).not.toHaveBeenCalled() + // p2-only failure must not bump metadata_last_run_at — phase 1 already succeeded, + // but versions/deps never refreshed, so the package must stay due expect(mockMarkMetadata).toHaveBeenCalledWith( qx, PURL, expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), + undefined, + false, ) }) @@ -224,6 +228,8 @@ describe('ingestOnePackagistMetadata', () => { qx, PURL, expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), + undefined, + false, ) }) diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index e41451357e..a6898bcbbc 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -151,7 +151,16 @@ export async function ingestOnePackagistMetadata( { purl: candidate.purl, statusCode: p2.error.statusCode, kind: p2.error.kind }, 'packagist metadata 4xx/malformed after fast retries — marking scanned and skipping', ) - await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(p2.error, p2.attempts)) + // Phase 1 already succeeded — only p2 (versions/deps) failed to refresh. Don't push + // metadata_last_run_at forward, or due-selection wrongly treats this package as + // "recently scanned" and skips it for the full refresh window despite stale p2 data. + await markPackagistMetadataScanned( + qx, + candidate.purl, + giveUpResult(p2.error, p2.attempts), + undefined, + false, + ) return } diff --git a/services/libs/data-access-layer/src/packages/packagistPackageState.ts b/services/libs/data-access-layer/src/packages/packagistPackageState.ts index 96991f0ef8..74f74b409d 100644 --- a/services/libs/data-access-layer/src/packages/packagistPackageState.ts +++ b/services/libs/data-access-layer/src/packages/packagistPackageState.ts @@ -26,15 +26,25 @@ export async function markPackagistMetadataScanned( purl: string, result: PackagistRunResult, metadataLastModified?: string | null, + // p2-only failures (phase 1 already succeeded) should NOT push the refresh watermark + // forward — versions/dependencies never actually refreshed, so the package must stay + // (or become) due again on the next run instead of sitting out the full refresh window. + bumpLastRunAt = true, ): Promise { await qx.result( `INSERT INTO packagist_package_state (purl, metadata_run_result, metadata_last_run_at, metadata_last_modified) - VALUES ($(purl), $(result)::jsonb, NOW(), $(metadataLastModified)) + VALUES ($(purl), $(result)::jsonb, CASE WHEN $(bumpLastRunAt) THEN NOW() ELSE NULL END, $(metadataLastModified)) ON CONFLICT (purl) DO UPDATE SET metadata_run_result = EXCLUDED.metadata_run_result, - metadata_last_run_at = EXCLUDED.metadata_last_run_at, + metadata_last_run_at = CASE WHEN $(bumpLastRunAt) THEN EXCLUDED.metadata_last_run_at + ELSE packagist_package_state.metadata_last_run_at END, metadata_last_modified = COALESCE(EXCLUDED.metadata_last_modified, packagist_package_state.metadata_last_modified)`, - { purl, result: JSON.stringify(result), metadataLastModified: metadataLastModified ?? null }, + { + purl, + result: JSON.stringify(result), + metadataLastModified: metadataLastModified ?? null, + bumpLastRunAt, + }, ) } From 77e6b6c895c7906667454670acd9cca55d354487 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 17:59:12 +0200 Subject: [PATCH 30/56] fix: guard packagist watermark against re-processing Signed-off-by: anilb --- .../packagist/__tests__/dueSelection.test.ts | 2 +- .../src/packagist/__tests__/ingest.test.ts | 41 ++++----- .../src/packagist/activities.ts | 85 ++++++++++++------- .../src/packages/packagistPackageState.ts | 54 +++++++++--- 4 files changed, 120 insertions(+), 62 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts index 31d6ada7a9..3bd1c2ea22 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts @@ -72,7 +72,7 @@ describe('markPackagistMetadataScanned', () => { asQx(qx), 'pkg:composer/a/x', { status: 'success', attempts: 1 }, - 'Wed, 01 Jul 2026 00:00:00 GMT', + { metadataLastModified: 'Wed, 01 Jul 2026 00:00:00 GMT' }, ) const [sql, params] = qx.result.mock.calls[0] diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts index b76ce330b5..07e5f36e53 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -75,6 +75,7 @@ const mockSeedInsert = vi.mocked(insertPackagistPackages) const qx = {} as QueryExecutor const PURL = 'pkg:composer/monolog/monolog' const RUN_DATE = '2026-07-15' +const SCHEDULED_AT = '2026-07-15T00:00:00.000Z' const statsJson = { package: { name: 'monolog/monolog', downloads: { total: 1000, monthly: 300, daily: 10 } }, @@ -113,7 +114,7 @@ describe('ingestOnePackagistMetadata', () => { it('audits phase 1 immediately and phase 2 separately, then marks scanned with the fresh Last-Modified', async () => { happyMocks() - await ingestOnePackagistMetadata(qx, candidate) + await ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) expect(mockPersistInfo).toHaveBeenCalledWith(qx, PURL, expect.anything()) expect(mockFetchP2).toHaveBeenCalledWith('monolog/monolog', 'Tue, 30 Jun 2026 00:00:00 GMT') @@ -127,7 +128,7 @@ describe('ingestOnePackagistMetadata', () => { qx, PURL, expect.objectContaining({ status: 'success', attempts: 1 }), - 'Wed, 01 Jul 2026 00:00:00 GMT', + { metadataLastModified: 'Wed, 01 Jul 2026 00:00:00 GMT' }, ) }) @@ -135,7 +136,7 @@ describe('ingestOnePackagistMetadata', () => { happyMocks() mockFetchP2.mockResolvedValue({ kind: 'NOT_MODIFIED' } as never) - await ingestOnePackagistMetadata(qx, candidate) + await ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) expect(mockPersistInfo).toHaveBeenCalled() expect(mockPersistMetadata).not.toHaveBeenCalled() @@ -144,7 +145,7 @@ describe('ingestOnePackagistMetadata', () => { qx, PURL, expect.objectContaining({ status: 'success' }), - null, + { metadataLastModified: null }, ) }) @@ -156,7 +157,7 @@ describe('ingestOnePackagistMetadata', () => { message: 'not found', } as never) - const p = ingestOnePackagistMetadata(qx, candidate) + const p = ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) await vi.runAllTimersAsync() await p @@ -167,6 +168,7 @@ describe('ingestOnePackagistMetadata', () => { qx, PURL, expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND', httpStatus: 404 }), + { notBefore: SCHEDULED_AT }, ) }) @@ -177,7 +179,7 @@ describe('ingestOnePackagistMetadata', () => { message: 'HTTP 500', } as never) - await expect(ingestOnePackagistMetadata(qx, candidate)).rejects.toThrow() + await expect(ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT)).rejects.toThrow() expect(mockMarkMetadata).not.toHaveBeenCalled() expect(mockFetchStats).toHaveBeenCalledTimes(1) }) @@ -192,7 +194,7 @@ describe('ingestOnePackagistMetadata', () => { message: 'not found', } as never) - const p = ingestOnePackagistMetadata(qx, candidate) + const p = ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) await vi.runAllTimersAsync() await p @@ -203,8 +205,7 @@ describe('ingestOnePackagistMetadata', () => { qx, PURL, expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), - undefined, - false, + { bumpLastRunAt: false, notBefore: SCHEDULED_AT }, ) }) @@ -219,7 +220,7 @@ describe('ingestOnePackagistMetadata', () => { message: 'not found', } as never) - const p = ingestOnePackagistMetadata(qx, candidate) + const p = ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) await vi.runAllTimersAsync() await p @@ -228,8 +229,7 @@ describe('ingestOnePackagistMetadata', () => { qx, PURL, expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), - undefined, - false, + { bumpLastRunAt: false, notBefore: SCHEDULED_AT }, ) }) @@ -238,7 +238,7 @@ describe('ingestOnePackagistMetadata', () => { mockPersistInfo.mockResolvedValue({ found: true, changedFields: ['packages.description'] }) mockFetchP2.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 502' } as never) - await expect(ingestOnePackagistMetadata(qx, candidate)).rejects.toThrow() + await expect(ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT)).rejects.toThrow() expect(mockMarkMetadata).not.toHaveBeenCalled() // phase-1 writes are already committed when the throw happens — a retry re-runs // phase 1 idempotently and reports no changes, so this audit event can't be deferred @@ -252,7 +252,7 @@ describe('ingestOnePackagist30dWindow', () => { mockFetchStats.mockResolvedValue(statsJson as never) mockPersist30d.mockResolvedValue(['downloads_last_30d.count', 'packages.downloads_last_30d']) - await ingestOnePackagist30dWindow(qx, PURL, RUN_DATE) + await ingestOnePackagist30dWindow(qx, PURL, RUN_DATE, SCHEDULED_AT) expect(mockPersist30d).toHaveBeenCalledWith(qx, PURL, 300, RUN_DATE) expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, [ @@ -274,7 +274,7 @@ describe('ingestOnePackagist30dWindow', () => { message: 'not found', } as never) - const p = ingestOnePackagist30dWindow(qx, PURL, RUN_DATE) + const p = ingestOnePackagist30dWindow(qx, PURL, RUN_DATE, SCHEDULED_AT) await vi.runAllTimersAsync() await p @@ -283,13 +283,14 @@ describe('ingestOnePackagist30dWindow', () => { qx, PURL, expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), + SCHEDULED_AT, ) }) it('throws on a transient result without marking processed', async () => { mockFetchStats.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 503' } as never) - await expect(ingestOnePackagist30dWindow(qx, PURL, RUN_DATE)).rejects.toThrow() + await expect(ingestOnePackagist30dWindow(qx, PURL, RUN_DATE, SCHEDULED_AT)).rejects.toThrow() expect(mockMark30d).not.toHaveBeenCalled() }) }) @@ -302,7 +303,7 @@ describe('ingestOnePackagistDailyDownload', () => { mockFetchStats.mockResolvedValue(statsJson as never) mockDaily.mockResolvedValue(['downloads_daily.date', 'downloads_daily.count']) - await ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE) + await ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE, SCHEDULED_AT) expect(mockDaily).toHaveBeenCalledWith(qx, '7', [{ day: RUN_DATE, downloads: 10 }]) expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, [ @@ -319,7 +320,7 @@ describe('ingestOnePackagistDailyDownload', () => { it('marks success without inserting or auditing when the registry reports no daily count', async () => { mockFetchStats.mockResolvedValue({ package: { name: 'monolog/monolog' } } as never) - await ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE) + await ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE, SCHEDULED_AT) expect(mockDaily).not.toHaveBeenCalled() expect(mockAudit).not.toHaveBeenCalled() @@ -333,7 +334,9 @@ describe('ingestOnePackagistDailyDownload', () => { it('throws on a transient result without marking processed', async () => { mockFetchStats.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 503' } as never) - await expect(ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE)).rejects.toThrow() + await expect( + ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE, SCHEDULED_AT), + ).rejects.toThrow() expect(mockMarkDaily).not.toHaveBeenCalled() }) }) diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index a6898bcbbc..83fa211af4 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -119,6 +119,9 @@ function giveUpResult(error: FetchError, attempts: number): PackagistRunResult { export async function ingestOnePackagistMetadata( qx: QueryExecutor, candidate: PackagistMetadataCandidate, + // This batch activity's own scheduledTimestampMs (stable across Temporal retries), + // passed through to every give-up write below — see MarkMetadataScannedOptions.notBefore. + scheduledAt: string, ): Promise { const name = packagistNameFromPurl(candidate.purl) @@ -132,7 +135,14 @@ export async function ingestOnePackagistMetadata( { purl: candidate.purl, statusCode: info.error.statusCode, kind: info.error.kind }, 'packagist package info 4xx/malformed after fast retries — marking scanned and skipping', ) - await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(info.error, info.attempts)) + await markPackagistMetadataScanned( + qx, + candidate.purl, + giveUpResult(info.error, info.attempts), + { + notBefore: scheduledAt, + }, + ) return } @@ -154,13 +164,10 @@ export async function ingestOnePackagistMetadata( // Phase 1 already succeeded — only p2 (versions/deps) failed to refresh. Don't push // metadata_last_run_at forward, or due-selection wrongly treats this package as // "recently scanned" and skips it for the full refresh window despite stale p2 data. - await markPackagistMetadataScanned( - qx, - candidate.purl, - giveUpResult(p2.error, p2.attempts), - undefined, - false, - ) + await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(p2.error, p2.attempts), { + bumpLastRunAt: false, + notBefore: scheduledAt, + }) return } @@ -180,13 +187,13 @@ export async function ingestOnePackagistMetadata( } await logAuditFieldChanges(qx, WORKER, candidate.purl, phase2ChangedFields) - // markPackagistMetadataScanned treats a null 4th arg the same as omitting it - // (COALESCEd against the stored value), so lastModified can always be passed. await markPackagistMetadataScanned( qx, candidate.purl, { status: 'success', attempts: p2.attempts }, - lastModified, + { + metadataLastModified: lastModified, + }, ) } @@ -195,6 +202,7 @@ export async function ingestOnePackagist30dWindow( qx: QueryExecutor, purl: string, runDate: string, + scheduledAt: string, ): Promise { const name = packagistNameFromPurl(purl) @@ -207,7 +215,7 @@ export async function ingestOnePackagist30dWindow( { purl, statusCode: info.error.statusCode, kind: info.error.kind }, 'packagist 30d downloads 4xx/malformed after fast retries — marking processed and skipping', ) - await markPackagist30dProcessed(qx, purl, giveUpResult(info.error, info.attempts)) + await markPackagist30dProcessed(qx, purl, giveUpResult(info.error, info.attempts), scheduledAt) return } @@ -226,6 +234,7 @@ export async function ingestOnePackagistDailyDownload( qx: QueryExecutor, candidate: PackagistDailyCandidate, runDate: string, + scheduledAt: string, ): Promise { const name = packagistNameFromPurl(candidate.purl) @@ -238,7 +247,12 @@ export async function ingestOnePackagistDailyDownload( { purl: candidate.purl, statusCode: info.error.statusCode, kind: info.error.kind }, 'packagist daily downloads 4xx/malformed after fast retries — marking processed and skipping', ) - await markPackagistDailyProcessed(qx, candidate.purl, giveUpResult(info.error, info.attempts)) + await markPackagistDailyProcessed( + qx, + candidate.purl, + giveUpResult(info.error, info.attempts), + scheduledAt, + ) return } @@ -332,6 +346,8 @@ export async function ingestPackagistMetadataBatch( if (candidates.length === 0) return const qx = await getPackagesDb() const attempt = Context.current().info.attempt + // Stable across every Temporal retry of this same batch — see notBefore below. + const scheduledAt = new Date(Context.current().info.scheduledTimestampMs).toISOString() // The merged lane starts every ingest with a DYNAMIC-endpoint fetch, so it is // bounded by that endpoint's 10-concurrent limit — not p2's 20. Running hotter @@ -340,13 +356,16 @@ export async function ingestPackagistMetadataBatch( candidates, attempt, statsConcurrency(), - (candidate) => ingestOnePackagistMetadata(qx, candidate), + (candidate) => ingestOnePackagistMetadata(qx, candidate, scheduledAt), (candidate, err) => - markPackagistMetadataScanned(qx, candidate.purl, { - status: 'error', - attempts: attempt, - message: String(err), - }), + markPackagistMetadataScanned( + qx, + candidate.purl, + { status: 'error', attempts: attempt, message: String(err) }, + // An item that already succeeded earlier in this same batch's retry sequence + // must not have that success overwritten by an unrelated re-processing failure. + { notBefore: scheduledAt }, + ), ) log.info({ count: candidates.length }, 'Ingested Packagist metadata batch') @@ -366,18 +385,20 @@ export async function ingestPackagist30dBatch(purls: string[], runDate: string): if (purls.length === 0) return const qx = await getPackagesDb() const attempt = Context.current().info.attempt + const scheduledAt = new Date(Context.current().info.scheduledTimestampMs).toISOString() await ingestPackagistItemsConcurrently( purls, attempt, statsConcurrency(), - (purl) => ingestOnePackagist30dWindow(qx, purl, runDate), + (purl) => ingestOnePackagist30dWindow(qx, purl, runDate, scheduledAt), (purl, err) => - markPackagist30dProcessed(qx, purl, { - status: 'error', - attempts: attempt, - message: String(err), - }), + markPackagist30dProcessed( + qx, + purl, + { status: 'error', attempts: attempt, message: String(err) }, + scheduledAt, + ), ) log.info({ count: purls.length }, 'Ingested Packagist 30d downloads batch') @@ -403,18 +424,20 @@ export async function ingestPackagistDailyBatch( if (candidates.length === 0) return const qx = await getPackagesDb() const attempt = Context.current().info.attempt + const scheduledAt = new Date(Context.current().info.scheduledTimestampMs).toISOString() await ingestPackagistItemsConcurrently( candidates, attempt, statsConcurrency(), - (candidate) => ingestOnePackagistDailyDownload(qx, candidate, runDate), + (candidate) => ingestOnePackagistDailyDownload(qx, candidate, runDate, scheduledAt), (candidate, err) => - markPackagistDailyProcessed(qx, candidate.purl, { - status: 'error', - attempts: attempt, - message: String(err), - }), + markPackagistDailyProcessed( + qx, + candidate.purl, + { status: 'error', attempts: attempt, message: String(err) }, + scheduledAt, + ), ) log.info({ count: candidates.length }, 'Ingested Packagist daily downloads batch') diff --git a/services/libs/data-access-layer/src/packages/packagistPackageState.ts b/services/libs/data-access-layer/src/packages/packagistPackageState.ts index 74f74b409d..3e6a24aaad 100644 --- a/services/libs/data-access-layer/src/packages/packagistPackageState.ts +++ b/services/libs/data-access-layer/src/packages/packagistPackageState.ts @@ -21,16 +21,28 @@ export interface PackagistDailyCandidate { packageId: string } +export interface MarkMetadataScannedOptions { + metadataLastModified?: string | null + // p2-only failures (phase 1 already succeeded) should NOT push the refresh watermark + // forward — versions/dependencies never actually refreshed, so the package must stay + // (or become) due again on the next run instead of sitting out the full refresh window. + bumpLastRunAt?: boolean + // Give-up (error) writes only: this activity's own scheduledTimestampMs (stable across + // Temporal retries of the same batch). Guards against a spurious re-processing of an + // item that already succeeded earlier in this same batch's retry sequence overwriting + // that success with an error — only blocks the write if the stored success happened at + // or after this activity was first scheduled, so a genuinely new failure on a later, + // unrelated run still always records normally. + notBefore?: string | null +} + export async function markPackagistMetadataScanned( qx: QueryExecutor, purl: string, result: PackagistRunResult, - metadataLastModified?: string | null, - // p2-only failures (phase 1 already succeeded) should NOT push the refresh watermark - // forward — versions/dependencies never actually refreshed, so the package must stay - // (or become) due again on the next run instead of sitting out the full refresh window. - bumpLastRunAt = true, + options: MarkMetadataScannedOptions = {}, ): Promise { + const { metadataLastModified = null, bumpLastRunAt = true, notBefore = null } = options await qx.result( `INSERT INTO packagist_package_state (purl, metadata_run_result, metadata_last_run_at, metadata_last_modified) VALUES ($(purl), $(result)::jsonb, CASE WHEN $(bumpLastRunAt) THEN NOW() ELSE NULL END, $(metadataLastModified)) @@ -38,12 +50,18 @@ export async function markPackagistMetadataScanned( metadata_run_result = EXCLUDED.metadata_run_result, metadata_last_run_at = CASE WHEN $(bumpLastRunAt) THEN EXCLUDED.metadata_last_run_at ELSE packagist_package_state.metadata_last_run_at END, - metadata_last_modified = COALESCE(EXCLUDED.metadata_last_modified, packagist_package_state.metadata_last_modified)`, + metadata_last_modified = COALESCE(EXCLUDED.metadata_last_modified, packagist_package_state.metadata_last_modified) + WHERE $(notBefore)::timestamptz IS NULL + OR NOT ( + packagist_package_state.metadata_run_result->>'status' = 'success' + AND packagist_package_state.metadata_last_run_at >= $(notBefore)::timestamptz + )`, { purl, result: JSON.stringify(result), - metadataLastModified: metadataLastModified ?? null, + metadataLastModified, bumpLastRunAt, + notBefore, }, ) } @@ -103,14 +121,21 @@ export async function markPackagist30dProcessed( qx: QueryExecutor, purl: string, result: PackagistRunResult, + // Give-up (error) writes only — see markPackagistMetadataScanned's notBefore for why. + notBefore: string | null = null, ): Promise { await qx.result( `INSERT INTO packagist_package_state (purl, downloads_30d_run_result, downloads_30d_last_run_at) VALUES ($(purl), $(result)::jsonb, NOW()) ON CONFLICT (purl) DO UPDATE SET downloads_30d_run_result = EXCLUDED.downloads_30d_run_result, - downloads_30d_last_run_at = EXCLUDED.downloads_30d_last_run_at`, - { purl, result: JSON.stringify(result) }, + downloads_30d_last_run_at = EXCLUDED.downloads_30d_last_run_at + WHERE $(notBefore)::timestamptz IS NULL + OR NOT ( + packagist_package_state.downloads_30d_run_result->>'status' = 'success' + AND packagist_package_state.downloads_30d_last_run_at >= $(notBefore)::timestamptz + )`, + { purl, result: JSON.stringify(result), notBefore }, ) } @@ -142,13 +167,20 @@ export async function markPackagistDailyProcessed( qx: QueryExecutor, purl: string, result: PackagistRunResult, + // Give-up (error) writes only — see markPackagistMetadataScanned's notBefore for why. + notBefore: string | null = null, ): Promise { await qx.result( `INSERT INTO packagist_package_state (purl, daily_downloads_run_result, daily_downloads_last_run_at) VALUES ($(purl), $(result)::jsonb, NOW()) ON CONFLICT (purl) DO UPDATE SET daily_downloads_run_result = EXCLUDED.daily_downloads_run_result, - daily_downloads_last_run_at = EXCLUDED.daily_downloads_last_run_at`, - { purl, result: JSON.stringify(result) }, + daily_downloads_last_run_at = EXCLUDED.daily_downloads_last_run_at + WHERE $(notBefore)::timestamptz IS NULL + OR NOT ( + packagist_package_state.daily_downloads_run_result->>'status' = 'success' + AND packagist_package_state.daily_downloads_last_run_at >= $(notBefore)::timestamptz + )`, + { purl, result: JSON.stringify(result), notBefore }, ) } From 666be44c2141b98dd67ba61b38ebefdd90ed0cf5 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 17:59:17 +0200 Subject: [PATCH 31/56] fix: preserve GitLab subgroup paths in canonicalizeRepoUrl Signed-off-by: anilb --- .../__tests__/canonicalizeRepoUrl.test.ts | 20 ++++++++++++++++ .../src/utils/canonicalizeRepoUrl.ts | 24 +++++++++++++++---- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts b/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts index 544c368b30..dab3844b21 100644 --- a/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts +++ b/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts @@ -29,6 +29,26 @@ describe('canonicalizeRepoUrl', () => { 'github', ], ['ssh://git@github.com:2222/foo/bar.git', 'https://github.com/foo/bar', 'github'], + [ + 'https://gitlab.com/group/subgroup/project', + 'https://gitlab.com/group/subgroup/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/subgroup/subsubgroup/project.git', + 'https://gitlab.com/group/subgroup/subsubgroup/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/project/-/tree/master/src', + 'https://gitlab.com/group/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/subgroup/project/-/blob/main/README.md', + 'https://gitlab.com/group/subgroup/project', + 'gitlab', + ], ])('canonicalizes %s', (input, expectedUrl, expectedHost) => { expect(canonicalizeRepoUrl(input)).toEqual({ url: expectedUrl, host: expectedHost }) }) diff --git a/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts b/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts index a308697a45..ff7ed43002 100644 --- a/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts +++ b/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts @@ -32,8 +32,10 @@ const CASE_INSENSITIVE_HOSTS = new Set(['github.com', 'gitlab.com']) * Shared across the registry sub-workers (npm, Maven, …) and the GitHub * enricher so `repos.url` keys never diverge per ADR 0001. Handles npm * shorthand (`github:owner/repo`, bare `owner/repo`), SSH scp form, `ssh://`, - * `git+`, `git://`, `www.`, and monorepo `/tree//` deep-links - * (only the first two path segments are kept). Returns null when the input + * `git+`, `git://`, `www.`, and monorepo deep-links: GitHub/Bitbucket's bare + * `/tree//` (only the first two path segments are kept) and + * GitLab's `/-/tree//` (kept segments run up to the `/-/`, + * preserving arbitrarily nested subgroups). Returns null when the input * cannot be reduced to an owner/name pair. */ export function canonicalizeRepoUrl(raw: string): CanonicalRepo | null { @@ -80,8 +82,22 @@ export function canonicalizeRepoUrl(raw: string): CanonicalRepo | null { if (segments.length < 2) return null const isKnownHost = hostname in HOST_ENUM - let ownerPath = isKnownHost ? [segments[0]] : segments.slice(0, -1) - let name = (isKnownHost ? segments[1] : segments[segments.length - 1]).replace(/\.git$/, '') + // GitLab uniquely supports arbitrarily nested subgroups (group/subgroup/.../project), + // unlike GitHub/Bitbucket's flat owner/repo. Its deep-link suffixes (tree, blob, issues, + // merge_requests, ...) are marked off by a `/-/` path segment rather than appended + // directly after the repo path, so split there instead of truncating to 2 segments. + let pathSegments: string[] + if (hostname === 'gitlab.com') { + const dashIdx = segments.indexOf('-') + pathSegments = dashIdx === -1 ? segments : segments.slice(0, dashIdx) + } else if (isKnownHost) { + pathSegments = segments.slice(0, 2) + } else { + pathSegments = segments + } + + let ownerPath = pathSegments.slice(0, -1) + let name = (pathSegments[pathSegments.length - 1] ?? '').replace(/\.git$/, '') if (!name || ownerPath.length === 0 || ownerPath.some((seg) => !seg)) return null if (CASE_INSENSITIVE_HOSTS.has(hostname)) { From 91c6edc645dffcf478189d7dd92e84be0784f604 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 17:59:22 +0200 Subject: [PATCH 32/56] fix: prune stale repo links, reconcile maintainers Signed-off-by: anilb --- .../__tests__/persistPackageInfo.test.ts | Bin 6958 -> 8542 bytes .../src/packagist/upsertPackageInfo.ts | 89 +++++++++++------- .../data-access-layer/src/packages/repos.ts | 19 ++-- 3 files changed, 66 insertions(+), 42 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts index 62de1d7b81567dc7ec3a2d6eec57fdf8d0176c8d..adf1c48f62b8ef7df4a79151c1be4cca80e7c857 100644 GIT binary patch delta 721 zcmYk4O=}ZD7{}3;Hf)nLrP5ky;i(pO7ea!CD4S>yK@U>#{Zxufcb_Cfc4yX^$=ji* zM~^asc<4t^)SShuH~j=2`~rS~9(?j5$zfr3+5Nx#{`=|L@7td*FFtSpOuq_UUTO%> z3y-VdnJPT$pc>riW7siL`Eaui50y-~#v5J9qx$$scK8dB8}7y%bxS_qmY z2r1Lr?m8Fjt-{_?5!_0}9sYD`3kTzOHOWk@{yR;KdM%u2jFdH}CIfi(_yBZ|o3Mo% zjGPZ(2idwk%@^$1*0%kT&)T~iI}1VJ+AsO{_G4z#e$VYKUw6qq2<$iKVvSnLIGT8Q zT2f9)iPgBsa~}D0NMskIA_(k(6bN1r_|V4G%$FHi++V5K4~3n5g`pH7k0~63gb^sZ zNHXTy5Dp?1DVe5(hq7amnt^@2e9x|~U$uV5vw!oMjoDYaG&|dKt4}V?Zgmw7kKn|2 zXnufl`R(b}OK8!IgeqnsmLrVO5PeJ-jTLJ6^XeUN14)O1Xv5F0_O LXm@y)+t2+21J?B9 delta 36 scmccTw9agU5({JP=0cWDY?ImeBqpyHmfQT1`#;lUd0w{74kB{A0QBt)5&!@I diff --git a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts index 91a7eb371a..775dd0a70f 100644 --- a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts +++ b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts @@ -14,7 +14,9 @@ import type { NormalizedPackagistStats } from './types' // Dynamic-endpoint persistence: packages fields + repo link for ALL packages, // maintainers only for critical ones. Download rows are NOT written here — they -// belong to the dedicated downloads-30d/daily lanes. +// belong to the dedicated downloads-30d/daily lanes. All writes share one +// transaction so a failure partway through can never leave the packages row, +// repo link, and maintainer set inconsistent with each other. export async function persistPackagistPackageInfo( qx: QueryExecutor, purl: string, @@ -24,7 +26,6 @@ export async function persistPackagistPackageInfo( // text columns reject; strip them before any field is persisted. stripNullBytesDeep(stats) - // Step 1: Update packages row const canonical = stats.repositoryUrl ? canonicalizeRepoUrl(stats.repositoryUrl) : null // Packagist's repository field is free-form/author-supplied. canonicalizeRepoUrl's // 'other' bucket also matches non-repo URLs (wikis, issue trackers, registry pages) @@ -32,43 +33,59 @@ export async function persistPackagistPackageInfo( // github.com/gitlab.com/bitbucket.org — rather than the shared utility's default, // which other callers (npm/maven/cargo) rely on staying permissive. const trustedRepo = canonical && canonical.host !== 'other' ? canonical : null - const result = await updatePackagistPackageStats(qx, { - purl, - description: stats.description, - declaredRepositoryUrl: stats.repositoryUrl, - repositoryUrl: trustedRepo?.url ?? null, - status: stats.status, - downloadsLast30d: stats.downloadsMonthly, - totalDownloads: stats.downloadsTotal, - dependentCount: stats.dependents, - }) - if (!result) { - return { found: false, changedFields: [] } - } + let found = false + const changedFields: string[] = [] + + await qx.tx(async (t) => { + // Step 1: Update packages row + const result = await updatePackagistPackageStats(t, { + purl, + description: stats.description, + declaredRepositoryUrl: stats.repositoryUrl, + repositoryUrl: trustedRepo?.url ?? null, + status: stats.status, + downloadsLast30d: stats.downloadsMonthly, + totalDownloads: stats.downloadsTotal, + dependentCount: stats.dependents, + }) - const { id, isCritical } = result - const changedFields = [...result.changedFields] + if (!result) return - // Step 2: Link the repo for ALL packages — 'declared'/0.8 is the manifest-declared - // convention shared by npm/pypi/maven/cargo. When there's no trusted repo (removed - // from the manifest, or no longer canonicalizable to a known host), clear any - // previously-declared link instead of leaving it pointing at a repo the package no - // longer declares. - if (trustedRepo) { - const repo = await getOrCreateRepoByUrl(qx, trustedRepo.url, trustedRepo.host) - const linkChanged = await upsertPackageRepo(qx, id, repo.id, 'declared', 0.8) - changedFields.push(...repo.changedFields, ...linkChanged) - } else { - const removedFields = await removeDeclaredPackageRepo(qx, id) - changedFields.push(...removedFields) - } + found = true + const { id, isCritical } = result + changedFields.push(...result.changedFields) - // Step 3: Maintainers only for critical packages - if (isCritical && stats.maintainers.length > 0) { - const maintainerChanges = await upsertPackageMaintainers(qx, id, stats.maintainers, 'packagist') - changedFields.push(...maintainerChanges) - } + // Step 2: Link the repo for ALL packages — 'declared'/0.8 is the manifest-declared + // convention shared by npm/pypi/maven/cargo. When there's no trusted repo (removed + // from the manifest, or no longer canonicalizable to a known host), or it now + // resolves to a different repo, clear any previously-declared link that no longer + // applies — package_repos' unique key is (package_id, repo_id), not (package_id, + // source), so upserting the new link alone would leave a stale one dangling. + if (trustedRepo) { + const repo = await getOrCreateRepoByUrl(t, trustedRepo.url, trustedRepo.host) + const linkChanged = await upsertPackageRepo(t, id, repo.id, 'declared', 0.8) + const removedFields = await removeDeclaredPackageRepo(t, id, repo.id) + changedFields.push(...repo.changedFields, ...linkChanged, ...removedFields) + } else { + const removedFields = await removeDeclaredPackageRepo(t, id) + changedFields.push(...removedFields) + } + + // Step 3: Maintainers only for critical packages. upsertPackageMaintainers always + // replaces the full stored set (including deleting rows for maintainers no longer + // reported), so it must run even when the registry now reports zero maintainers — + // skipping it on an empty list would leave stale maintainers attached forever. + if (isCritical) { + const maintainerChanges = await upsertPackageMaintainers( + t, + id, + stats.maintainers, + 'packagist', + ) + changedFields.push(...maintainerChanges) + } + }) - return { found: true, changedFields } + return { found, changedFields } } diff --git a/services/libs/data-access-layer/src/packages/repos.ts b/services/libs/data-access-layer/src/packages/repos.ts index 921053891d..362415b8ac 100644 --- a/services/libs/data-access-layer/src/packages/repos.ts +++ b/services/libs/data-access-layer/src/packages/repos.ts @@ -32,17 +32,24 @@ export async function getOrCreateRepoByUrl( return { id: row.id, changedFields: [] } } -// Removes a package's previous 'declared' repo link when its manifest no longer resolves -// to a trusted repo (the field was removed, or no longer canonicalizes). Only touches -// 'declared' — other sources (deps_dev, heuristic, manual) are owned by different -// pipelines and left alone. +// Removes a package's previous 'declared' repo link(s) when its manifest no longer +// resolves to a trusted repo (the field was removed, or no longer canonicalizes), or +// now resolves to a different one (pass exceptRepoId to keep only that fresh link — +// package_repos' unique key is (package_id, repo_id), not (package_id, source), so a +// plain upsert of the new link never removes a stale one pointing elsewhere). Only +// touches 'declared' — other sources (deps_dev, heuristic, manual) are owned by +// different pipelines and left alone. export async function removeDeclaredPackageRepo( qx: QueryExecutor, packageId: string, + exceptRepoId?: string, ): Promise { const rowCount = await qx.result( - `DELETE FROM package_repos WHERE package_id = $(packageId)::bigint AND source = 'declared'`, - { packageId }, + `DELETE FROM package_repos + WHERE package_id = $(packageId)::bigint + AND source = 'declared' + AND ($(exceptRepoId)::bigint IS NULL OR repo_id <> $(exceptRepoId)::bigint)`, + { packageId, exceptRepoId: exceptRepoId ?? null }, ) return rowCount > 0 ? ['package_repos.repo_id'] : [] } From 62519094eed87e6f623f2fc9ec784f1381f95dd1 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 17:59:26 +0200 Subject: [PATCH 33/56] fix: wrap packagist metadata writes in a transaction Signed-off-by: anilb --- .../__tests__/persistMetadata.test.ts | 4 +- .../src/packagist/upsertMetadata.ts | 130 ++++++++++-------- 2 files changed, 72 insertions(+), 62 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts index 8d89e048a2..de88e9f58f 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts @@ -23,7 +23,9 @@ const mockVersions = vi.mocked(upsertPackagistVersions) const mockIds = vi.mocked(getPackagistPackageIdsByNames) const mockDeps = vi.mocked(reconcileVersionDependencies) -const qx = {} as QueryExecutor +const qx = { + tx: vi.fn((cb: (t: QueryExecutor) => Promise) => cb(qx)), +} as unknown as QueryExecutor const PURL = 'pkg:composer/monolog/monolog' const expanded: PackagistExpandedVersion[] = [ diff --git a/services/apps/packages_worker/src/packagist/upsertMetadata.ts b/services/apps/packages_worker/src/packagist/upsertMetadata.ts index d2dfab9286..87114b5dc9 100644 --- a/services/apps/packages_worker/src/packagist/upsertMetadata.ts +++ b/services/apps/packages_worker/src/packagist/upsertMetadata.ts @@ -28,77 +28,85 @@ export async function persistPackagistMetadata( const { versionRows, latestVersion, firstReleaseAt, latestReleaseAt, licenses, homepage } = buildPackagistVersionRows(expanded) - const agg = await updatePackagistVersionAggregates(qx, purl, { - versionsCount: versionRows.length, - latestVersion, - firstReleaseAt, - latestReleaseAt, - licenses, - homepage, - }) + let found = false + const changedFields: string[] = [] + let unresolvedDependencyTargets = 0 - if (!agg) { - return { found: false, changedFields: [], unresolvedDependencyTargets: 0 } - } + // One transaction for the whole p2 write path: the aggregates update, version + // upsert + is_latest cleanup, and dependency reconcile (delete stale + upsert + // current) are each multi-statement on their own — sharing one tx means a failure + // partway through can never leave versions/dependencies half-refreshed. + await qx.tx(async (t) => { + const agg = await updatePackagistVersionAggregates(t, purl, { + versionsCount: versionRows.length, + latestVersion, + firstReleaseAt, + latestReleaseAt, + licenses, + homepage, + }) - const changedFields = [...agg.changedFields] + if (!agg) return - let versionIds: Array<{ number: string; id: string }> = [] - if (versionRows.length > 0) { - const versionResult = await upsertPackagistVersions(qx, agg.id, versionRows, latestVersion) - changedFields.push(...versionResult.changedFields) - versionIds = versionResult.versionIds - } + found = true + changedFields.push(...agg.changedFields) - // One pass over the tagged versions collects target names and provisional edges; - // target ids are then resolved in a single batch query. - const versionMap = new Map(versionIds.map((v) => [v.number, v.id])) - const targetNames = new Set() - const pending: Array<{ versionId: string; dep: PackagistDependency }> = [] + let versionIds: Array<{ number: string; id: string }> = [] + if (versionRows.length > 0) { + const versionResult = await upsertPackagistVersions(t, agg.id, versionRows, latestVersion) + changedFields.push(...versionResult.changedFields) + versionIds = versionResult.versionIds + } - for (const v of expanded) { - if (isPackagistDevVersion(v.version, v.version_normalized)) continue - const versionId = versionMap.get(v.version) - if (!versionId) continue + // One pass over the tagged versions collects target names and provisional edges; + // target ids are then resolved in a single batch query. + const versionMap = new Map(versionIds.map((v) => [v.number, v.id])) + const targetNames = new Set() + const pending: Array<{ versionId: string; dep: PackagistDependency }> = [] - for (const dep of extractVersionDependencies(v)) { - targetNames.add(dep.name) - pending.push({ versionId, dep }) + for (const v of expanded) { + if (isPackagistDevVersion(v.version, v.version_normalized)) continue + const versionId = versionMap.get(v.version) + if (!versionId) continue + + for (const dep of extractVersionDependencies(v)) { + targetNames.add(dep.name) + pending.push({ versionId, dep }) + } } - } - // Reconciled for every version being refreshed (not just ones with a resolved - // dependency this pass) — a version whose manifest drops a requirement, or all of - // them, must have its stale package_dependencies rows removed, not just the newly - // declared ones upserted. - let unresolvedDependencyTargets = 0 - if (versionIds.length > 0) { - const edges: VersionDependencyEdge[] = [] - if (pending.length > 0) { - const idMap = await getPackagistPackageIdsByNames(qx, Array.from(targetNames)) - for (const { versionId, dep } of pending) { - const dependsOnId = idMap.get(dep.name) - if (!dependsOnId) { - unresolvedDependencyTargets++ - continue + // Reconciled for every version being refreshed (not just ones with a resolved + // dependency this pass) — a version whose manifest drops a requirement, or all of + // them, must have its stale package_dependencies rows removed, not just the newly + // declared ones upserted. + if (versionIds.length > 0) { + const edges: VersionDependencyEdge[] = [] + if (pending.length > 0) { + const idMap = await getPackagistPackageIdsByNames(t, Array.from(targetNames)) + for (const { versionId, dep } of pending) { + const dependsOnId = idMap.get(dep.name) + if (!dependsOnId) { + unresolvedDependencyTargets++ + continue + } + edges.push({ + packageId: agg.id, + versionId, + dependsOnId, + constraint: dep.constraint, + kind: dep.kind, + }) } - edges.push({ - packageId: agg.id, - versionId, - dependsOnId, - constraint: dep.constraint, - kind: dep.kind, - }) } - } - const depChanges = await reconcileVersionDependencies( - qx, - versionIds.map((v) => v.id), - edges, - ) - changedFields.push(...depChanges) - } + const depChanges = await reconcileVersionDependencies( + t, + versionIds.map((v) => v.id), + edges, + ) + changedFields.push(...depChanges) + } + }) - return { found: true, changedFields, unresolvedDependencyTargets } + return { found, changedFields, unresolvedDependencyTargets } } From dc14b301d8a01f29974922d7097d21f9db7ce2c7 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 17:59:31 +0200 Subject: [PATCH 34/56] fix: make packagist 30d window insert atomic Signed-off-by: anilb --- .../src/packagist/__tests__/downloads.test.ts | 23 ++++------ .../src/packagist/downloads.ts | 18 ++++---- .../src/packages/downloadsLast30d.ts | 42 +++++++++++++++++++ 3 files changed, 58 insertions(+), 25 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts b/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts index 9e10e1db07..cb2885b174 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts @@ -1,27 +1,21 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -import { - getExistingLast30dEndDates, - upsertLast30dDownload, -} from '@crowd/data-access-layer/src/packages' +import { insertLast30dDownloadIfAbsent } from '@crowd/data-access-layer/src/packages' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { monthlyWindowFor, persistPackagist30dWindow } from '../downloads' vi.mock('@crowd/data-access-layer/src/packages', () => ({ - getExistingLast30dEndDates: vi.fn().mockResolvedValue([]), - upsertLast30dDownload: vi.fn().mockResolvedValue([]), + insertLast30dDownloadIfAbsent: vi.fn().mockResolvedValue([]), })) -const mockExistingWindows = vi.mocked(getExistingLast30dEndDates) -const mockWindow = vi.mocked(upsertLast30dDownload) +const mockWindow = vi.mocked(insertLast30dDownloadIfAbsent) const qx = {} as QueryExecutor const PURL = 'pkg:composer/monolog/monolog' beforeEach(() => { vi.clearAllMocks() - mockExistingWindows.mockResolvedValue([]) }) // Observed rolling window anchored on the 1st of the run month, @@ -49,30 +43,31 @@ describe('monthlyWindowFor', () => { // The monthly downloads-30d lane: one window row per purl per month, mirrored to // packages.downloads_last_30d (npm parity). A window already recorded for the month // is never overwritten — the value is the observation closest to the boundary. +// insertLast30dDownloadIfAbsent does the presence check and the write atomically, so +// there is no separate "does it exist" call to assert on here — a race is exercised +// directly against Postgres in downloadsLast30d's own DAL-level coverage. describe('persistPackagist30dWindow', () => { it('writes the window with the mirror and returns the changed fields when the month has no row yet', async () => { mockWindow.mockResolvedValue(['downloads_last_30d.count', 'packages.downloads_last_30d']) const changedFields = await persistPackagist30dWindow(qx, PURL, 300, '2026-07-01') - expect(mockExistingWindows).toHaveBeenCalledWith(qx, PURL, '2026-07-01', '2026-07-01') expect(mockWindow).toHaveBeenCalledWith(qx, PURL, '2026-06-01', '2026-07-01', 300, true) expect(changedFields).toEqual(['downloads_last_30d.count', 'packages.downloads_last_30d']) }) - it('does not overwrite an existing window for the month, and returns no changed fields', async () => { - mockExistingWindows.mockResolvedValue(['2026-07-01']) + it('returns no changed fields when a window for the month already exists', async () => { + mockWindow.mockResolvedValue([]) const changedFields = await persistPackagist30dWindow(qx, PURL, 300, '2026-07-15') - expect(mockWindow).not.toHaveBeenCalled() + expect(mockWindow).toHaveBeenCalledWith(qx, PURL, '2026-06-01', '2026-07-01', 300, true) expect(changedFields).toEqual([]) }) it('writes nothing and returns no changed fields when the registry reported no monthly count', async () => { const changedFields = await persistPackagist30dWindow(qx, PURL, null, '2026-07-01') - expect(mockExistingWindows).not.toHaveBeenCalled() expect(mockWindow).not.toHaveBeenCalled() expect(changedFields).toEqual([]) }) diff --git a/services/apps/packages_worker/src/packagist/downloads.ts b/services/apps/packages_worker/src/packagist/downloads.ts index 5a4af0b820..1034302499 100644 --- a/services/apps/packages_worker/src/packagist/downloads.ts +++ b/services/apps/packages_worker/src/packagist/downloads.ts @@ -1,7 +1,4 @@ -import { - getExistingLast30dEndDates, - upsertLast30dDownload, -} from '@crowd/data-access-layer/src/packages' +import { insertLast30dDownloadIfAbsent } from '@crowd/data-access-layer/src/packages' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' // Compute the monthly window for observed rolling 30d downloads. @@ -21,8 +18,11 @@ export function monthlyWindowFor(runDate: string): { startDate: string; endDate: // One window row per purl per month, mirrored to packages.downloads_last_30d // (npm parity). A window already recorded for the month is never overwritten — -// the value is the observation closest to the boundary. Returns the changed -// fields (like the sibling persist*/upsert* functions) — the caller audits them. +// the value is the observation closest to the boundary. insertLast30dDownloadIfAbsent +// does the presence check and the insert (+ mirror) atomically in one statement, so +// concurrent runs for the same purl+month can't both pass a check-then-insert gap and +// have the later one clobber the earlier one's count. Returns the changed fields (like +// the sibling persist*/upsert* functions) — the caller audits them. export async function persistPackagist30dWindow( qx: QueryExecutor, purl: string, @@ -32,9 +32,5 @@ export async function persistPackagist30dWindow( if (monthly == null) return [] const { startDate, endDate } = monthlyWindowFor(runDate) - const existing = await getExistingLast30dEndDates(qx, purl, endDate, endDate) - if (existing.length === 0) { - return upsertLast30dDownload(qx, purl, startDate, endDate, monthly, true) - } - return [] + return insertLast30dDownloadIfAbsent(qx, purl, startDate, endDate, monthly, true) } diff --git a/services/libs/data-access-layer/src/packages/downloadsLast30d.ts b/services/libs/data-access-layer/src/packages/downloadsLast30d.ts index f27e746af3..6189d4bf2b 100644 --- a/services/libs/data-access-layer/src/packages/downloadsLast30d.ts +++ b/services/libs/data-access-layer/src/packages/downloadsLast30d.ts @@ -133,6 +133,48 @@ export async function markLast30dHistoryBackfilled( ) } +// Atomic "first observation wins" insert: the window row and its packages mirror are +// written by one statement via ON CONFLICT (purl, end_date) DO NOTHING, so concurrent +// callers for the same purl+month (overlapping Temporal retries, or two lanes racing) +// can never both pass a check-then-insert gap and have the later one clobber the +// earlier one's count — only the writer that actually inserts the row also mirrors. +export async function insertLast30dDownloadIfAbsent( + qx: QueryExecutor, + purl: string, + startDate: string, + endDate: string, + count: number, + mirrorToPackages: boolean, +): Promise { + const row: { inserted: boolean; mirrored: boolean } = await qx.selectOne( + `WITH ins AS ( + INSERT INTO downloads_last_30d (purl, start_date, end_date, count, created_at, updated_at) + VALUES ($(purl), $(startDate)::date, $(endDate)::date, $(count), NOW(), NOW()) + ON CONFLICT (purl, end_date) DO NOTHING + RETURNING purl, count + ), + pkg AS ( + UPDATE packages p + SET downloads_last_30d = ins.count, + last_synced_at = NOW() + FROM ins + WHERE p.purl = ins.purl + AND $(mirrorToPackages) + AND p.downloads_last_30d IS DISTINCT FROM ins.count + RETURNING p.id + ) + SELECT + EXISTS (SELECT 1 FROM ins) AS inserted, + EXISTS (SELECT 1 FROM pkg) AS mirrored`, + { purl, startDate, endDate, count, mirrorToPackages }, + ) + + if (!row.inserted) return [] + const changed = ['downloads_last_30d.start_date', 'downloads_last_30d.count'] + if (row.mirrored) changed.push('packages.downloads_last_30d') + return changed +} + export async function upsertLast30dDownload( qx: QueryExecutor, purl: string, From 1b515413f989d2768d088dbe20b4dfd782101ce1 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 18:25:17 +0200 Subject: [PATCH 35/56] fix: handle legacy GitLab deep links with no /-/ marker Signed-off-by: anilb --- .../__tests__/canonicalizeRepoUrl.test.ts | 11 +++++++ .../src/utils/canonicalizeRepoUrl.ts | 30 +++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts b/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts index dab3844b21..a21906bf80 100644 --- a/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts +++ b/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts @@ -49,6 +49,17 @@ describe('canonicalizeRepoUrl', () => { 'https://gitlab.com/group/subgroup/project', 'gitlab', ], + [ + // Pre-2018 GitLab / shorthand copies: no `/-/` marker ahead of the deep-link. + 'https://gitlab.com/group/project/tree/master/src', + 'https://gitlab.com/group/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/subgroup/project/blob/main/README.md', + 'https://gitlab.com/group/subgroup/project', + 'gitlab', + ], ])('canonicalizes %s', (input, expectedUrl, expectedHost) => { expect(canonicalizeRepoUrl(input)).toEqual({ url: expectedUrl, host: expectedHost }) }) diff --git a/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts b/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts index ff7ed43002..58dfce174a 100644 --- a/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts +++ b/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts @@ -24,6 +24,22 @@ const HOST_ENUM: Record = { // same repo never produces two distinct repos.url keys. const CASE_INSENSITIVE_HOSTS = new Set(['github.com', 'gitlab.com']) +// Pre-2018 GitLab URLs (and shorthand copies of them still circulating) mark deep-links +// the same way GitHub does — appended directly after the project path, with no `/-/` +// separator. GitLab reserves these route words at the project-slug position precisely so +// they can never collide with a real project name (docs: user/reserved_names), so treating +// the first one as a deep-link boundary is safe even for arbitrarily nested subgroups. +const GITLAB_LEGACY_ROUTE_SEGMENTS = new Set([ + 'tree', + 'blob', + 'commits', + 'commit', + 'compare', + 'issues', + 'merge_requests', + 'wikis', +]) + /** * Canonicalize a source-repository URL to `{ url, host }` where url is * `https:////` and host is the coarse classification stored @@ -35,8 +51,10 @@ const CASE_INSENSITIVE_HOSTS = new Set(['github.com', 'gitlab.com']) * `git+`, `git://`, `www.`, and monorepo deep-links: GitHub/Bitbucket's bare * `/tree//` (only the first two path segments are kept) and * GitLab's `/-/tree//` (kept segments run up to the `/-/`, - * preserving arbitrarily nested subgroups). Returns null when the input - * cannot be reduced to an owner/name pair. + * preserving arbitrarily nested subgroups) — plus a legacy fallback for pre-2018 + * GitLab links with no `/-/` marker, cut at the first reserved route keyword + * (`tree`, `blob`, `issues`, ...) instead. Returns null when the input cannot be + * reduced to an owner/name pair. */ export function canonicalizeRepoUrl(raw: string): CanonicalRepo | null { let s = raw.trim().replace(/#.*$/, '') @@ -89,7 +107,13 @@ export function canonicalizeRepoUrl(raw: string): CanonicalRepo | null { let pathSegments: string[] if (hostname === 'gitlab.com') { const dashIdx = segments.indexOf('-') - pathSegments = dashIdx === -1 ? segments : segments.slice(0, dashIdx) + // Legacy fallback: a route keyword with no `/-/` ahead of it (index 2+, so at least + // group + project survive) also marks the boundary — whichever comes first wins. + const legacyIdx = segments.findIndex( + (seg, i) => i >= 2 && GITLAB_LEGACY_ROUTE_SEGMENTS.has(seg), + ) + const cutIdx = [dashIdx, legacyIdx].filter((i) => i !== -1).sort((a, b) => a - b)[0] + pathSegments = cutIdx === undefined ? segments : segments.slice(0, cutIdx) } else if (isKnownHost) { pathSegments = segments.slice(0, 2) } else { From 2fa15bf4da5cb2cd5d8364727047fc7e1361882f Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 18:33:41 +0200 Subject: [PATCH 36/56] fix: audit packagist writes inside their own transaction Signed-off-by: anilb --- .../src/packagist/__tests__/downloads.test.ts | 24 ++++++++++--- .../src/packagist/__tests__/ingest.test.ts | 36 +++++++++---------- .../__tests__/persistMetadata.test.ts | 11 ++++++ .../__tests__/persistPackageInfo.test.ts | 11 ++++++ .../src/packagist/activities.ts | 35 +++++++++--------- .../src/packagist/downloads.ts | 24 ++++++++++--- .../src/packagist/upsertMetadata.ts | 14 +++++--- .../src/packagist/upsertPackageInfo.ts | 13 +++++-- 8 files changed, 117 insertions(+), 51 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts b/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts index cb2885b174..6447b75c90 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts @@ -1,17 +1,24 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -import { insertLast30dDownloadIfAbsent } from '@crowd/data-access-layer/src/packages' +import { + insertLast30dDownloadIfAbsent, + logAuditFieldChanges, +} from '@crowd/data-access-layer/src/packages' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { monthlyWindowFor, persistPackagist30dWindow } from '../downloads' vi.mock('@crowd/data-access-layer/src/packages', () => ({ insertLast30dDownloadIfAbsent: vi.fn().mockResolvedValue([]), + logAuditFieldChanges: vi.fn(), })) const mockWindow = vi.mocked(insertLast30dDownloadIfAbsent) +const mockAudit = vi.mocked(logAuditFieldChanges) -const qx = {} as QueryExecutor +const qx = { + tx: vi.fn((cb: (t: QueryExecutor) => Promise) => cb(qx)), +} as unknown as QueryExecutor const PURL = 'pkg:composer/monolog/monolog' beforeEach(() => { @@ -45,23 +52,29 @@ describe('monthlyWindowFor', () => { // is never overwritten — the value is the observation closest to the boundary. // insertLast30dDownloadIfAbsent does the presence check and the write atomically, so // there is no separate "does it exist" call to assert on here — a race is exercised -// directly against Postgres in downloadsLast30d's own DAL-level coverage. +// directly against Postgres in downloadsLast30d's own DAL-level coverage. The audit +// record shares the same transaction as the write. describe('persistPackagist30dWindow', () => { - it('writes the window with the mirror and returns the changed fields when the month has no row yet', async () => { + it('writes the window with the mirror, audits it, and returns the changed fields when the month has no row yet', async () => { mockWindow.mockResolvedValue(['downloads_last_30d.count', 'packages.downloads_last_30d']) const changedFields = await persistPackagist30dWindow(qx, PURL, 300, '2026-07-01') expect(mockWindow).toHaveBeenCalledWith(qx, PURL, '2026-06-01', '2026-07-01', 300, true) + expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, [ + 'downloads_last_30d.count', + 'packages.downloads_last_30d', + ]) expect(changedFields).toEqual(['downloads_last_30d.count', 'packages.downloads_last_30d']) }) - it('returns no changed fields when a window for the month already exists', async () => { + it('returns no changed fields and does not audit when a window for the month already exists', async () => { mockWindow.mockResolvedValue([]) const changedFields = await persistPackagist30dWindow(qx, PURL, 300, '2026-07-15') expect(mockWindow).toHaveBeenCalledWith(qx, PURL, '2026-06-01', '2026-07-01', 300, true) + expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, []) expect(changedFields).toEqual([]) }) @@ -69,6 +82,7 @@ describe('persistPackagist30dWindow', () => { const changedFields = await persistPackagist30dWindow(qx, PURL, null, '2026-07-01') expect(mockWindow).not.toHaveBeenCalled() + expect(mockAudit).not.toHaveBeenCalled() expect(changedFields).toEqual([]) }) }) diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts index 07e5f36e53..086021a644 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -72,7 +72,9 @@ const mockFetchList = vi.mocked(fetchPackagistPackageList) const mockParseList = vi.mocked(parsePackagistPackageList) const mockSeedInsert = vi.mocked(insertPackagistPackages) -const qx = {} as QueryExecutor +const qx = { + tx: vi.fn((cb: (t: QueryExecutor) => Promise) => cb(qx)), +} as unknown as QueryExecutor const PURL = 'pkg:composer/monolog/monolog' const RUN_DATE = '2026-07-15' const SCHEDULED_AT = '2026-07-15T00:00:00.000Z' @@ -111,7 +113,7 @@ describe('ingestOnePackagistMetadata', () => { }) } - it('audits phase 1 immediately and phase 2 separately, then marks scanned with the fresh Last-Modified', async () => { + it('persists phase 1 immediately and phase 2 separately, then marks scanned with the fresh Last-Modified', async () => { happyMocks() await ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) @@ -120,10 +122,9 @@ describe('ingestOnePackagistMetadata', () => { expect(mockFetchP2).toHaveBeenCalledWith('monolog/monolog', 'Tue, 30 Jun 2026 00:00:00 GMT') expect(mockExpand).toHaveBeenCalledWith(minified) expect(mockPersistMetadata).toHaveBeenCalledWith(qx, PURL, expanded) - // Two separate audit rows, not one merged call — phase 1 is logged as soon as it's - // committed, before the p2 fetch (which can throw) ever runs. - expect(mockAudit).toHaveBeenNthCalledWith(1, qx, 'packagist', PURL, ['packages.description']) - expect(mockAudit).toHaveBeenNthCalledWith(2, qx, 'packagist', PURL, ['versions.number']) + // Each phase audits its own writes atomically inside its own persist* call (see + // persistPackageInfo.test.ts / persistMetadata.test.ts) — phase 1 is committed and + // audited before the p2 fetch (which can throw) ever runs. expect(mockMarkMetadata).toHaveBeenCalledWith( qx, PURL, @@ -209,8 +210,9 @@ describe('ingestOnePackagistMetadata', () => { ) }) - it('audits phase-1 changes even when the p2 fetch gives up', async () => { - // the dynamic-endpoint writes are committed by then — their audit rows must not be dropped + it('still persists (and self-audits) phase 1 even when the p2 fetch gives up', async () => { + // the dynamic-endpoint writes are committed and audited by persistPackagistPackageInfo + // itself before the p2 fetch (which can throw) ever runs — see persistPackageInfo.test.ts vi.useFakeTimers() mockFetchStats.mockResolvedValue(statsJson as never) mockPersistInfo.mockResolvedValue({ found: true, changedFields: ['packages.description'] }) @@ -224,7 +226,7 @@ describe('ingestOnePackagistMetadata', () => { await vi.runAllTimersAsync() await p - expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, ['packages.description']) + expect(mockPersistInfo).toHaveBeenCalledWith(qx, PURL, expect.anything()) expect(mockMarkMetadata).toHaveBeenCalledWith( qx, PURL, @@ -233,32 +235,30 @@ describe('ingestOnePackagistMetadata', () => { ) }) - it('throws on a transient p2 result without marking scanned, but still audits phase 1', async () => { + it('throws on a transient p2 result without marking scanned, but phase 1 already persisted', async () => { mockFetchStats.mockResolvedValue(statsJson as never) mockPersistInfo.mockResolvedValue({ found: true, changedFields: ['packages.description'] }) mockFetchP2.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 502' } as never) await expect(ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT)).rejects.toThrow() expect(mockMarkMetadata).not.toHaveBeenCalled() - // phase-1 writes are already committed when the throw happens — a retry re-runs - // phase 1 idempotently and reports no changes, so this audit event can't be deferred - expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, ['packages.description']) + // phase-1 writes (and their audit row) are already committed by + // persistPackagistPackageInfo when the throw happens — a retry re-runs phase 1 + // idempotently and reports no changes, so nothing here can lose that audit event + expect(mockPersistInfo).toHaveBeenCalledWith(qx, PURL, expect.anything()) }) }) // The monthly downloads-30d lane: dynamic fetch, window row only. describe('ingestOnePackagist30dWindow', () => { - it('persists the observed rolling window, audits the change, and marks the run processed', async () => { + it('persists the observed rolling window (self-audited) and marks the run processed', async () => { mockFetchStats.mockResolvedValue(statsJson as never) mockPersist30d.mockResolvedValue(['downloads_last_30d.count', 'packages.downloads_last_30d']) await ingestOnePackagist30dWindow(qx, PURL, RUN_DATE, SCHEDULED_AT) expect(mockPersist30d).toHaveBeenCalledWith(qx, PURL, 300, RUN_DATE) - expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, [ - 'downloads_last_30d.count', - 'packages.downloads_last_30d', - ]) + // persistPackagist30dWindow audits its own write atomically — see downloads.test.ts expect(mockMark30d).toHaveBeenCalledWith( qx, PURL, diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts index de88e9f58f..1665157b83 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { getPackagistPackageIdsByNames, + logAuditFieldChanges, reconcileVersionDependencies, updatePackagistVersionAggregates, upsertPackagistVersions, @@ -16,12 +17,14 @@ vi.mock('@crowd/data-access-layer/src/packages', () => ({ upsertPackagistVersions: vi.fn(), getPackagistPackageIdsByNames: vi.fn(), reconcileVersionDependencies: vi.fn().mockResolvedValue([]), + logAuditFieldChanges: vi.fn(), })) const mockAggregates = vi.mocked(updatePackagistVersionAggregates) const mockVersions = vi.mocked(upsertPackagistVersions) const mockIds = vi.mocked(getPackagistPackageIdsByNames) const mockDeps = vi.mocked(reconcileVersionDependencies) +const mockAudit = vi.mocked(logAuditFieldChanges) const qx = { tx: vi.fn((cb: (t: QueryExecutor) => Promise) => cb(qx)), @@ -120,6 +123,13 @@ describe('persistPackagistMetadata', () => { expect(result.unresolvedDependencyTargets).toBe(0) // dependency-edge changes must reach the audit log alongside aggregates/versions expect(result.changedFields).toContain('package_dependencies.depends_on_id') + // audited atomically inside the same transaction as the writes above + expect(mockAudit).toHaveBeenCalledWith( + qx, + 'packagist', + PURL, + expect.arrayContaining(['packages.latest_version', 'package_dependencies.depends_on_id']), + ) }) it('skips and counts dependency targets that do not resolve to a packages row', async () => { @@ -170,6 +180,7 @@ describe('persistPackagistMetadata', () => { expect(mockVersions).not.toHaveBeenCalled() expect(mockIds).not.toHaveBeenCalled() expect(mockDeps).not.toHaveBeenCalled() + expect(mockAudit).not.toHaveBeenCalled() }) it('handles a dev-branches-only package: aggregates written, no version or dependency writes', async () => { diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts index adf1c48f62..ac58470779 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { getOrCreateRepoByUrl, + logAuditFieldChanges, removeDeclaredPackageRepo, updatePackagistPackageStats, upsertPackageMaintainers, @@ -18,6 +19,7 @@ vi.mock('@crowd/data-access-layer/src/packages', () => ({ getOrCreateRepoByUrl: vi.fn(), upsertPackageRepo: vi.fn().mockResolvedValue([]), removeDeclaredPackageRepo: vi.fn().mockResolvedValue([]), + logAuditFieldChanges: vi.fn(), })) const mockUpdate = vi.mocked(updatePackagistPackageStats) @@ -25,6 +27,7 @@ const mockMaintainers = vi.mocked(upsertPackageMaintainers) const mockRepoGet = vi.mocked(getOrCreateRepoByUrl) const mockRepoLink = vi.mocked(upsertPackageRepo) const mockRepoRemove = vi.mocked(removeDeclaredPackageRepo) +const mockAudit = vi.mocked(logAuditFieldChanges) const qx = { tx: vi.fn((cb: (t: QueryExecutor) => Promise) => cb(qx)), @@ -82,6 +85,13 @@ describe('persistPackagistPackageInfo', () => { expect(result.changedFields).toContain('packages.description') // maintainer changes must reach the audit log too, matching the npm/pypi paths expect(result.changedFields).toContain('maintainers.display_name') + // audited atomically inside the same transaction as the writes above + expect(mockAudit).toHaveBeenCalledWith( + qx, + 'packagist', + PURL, + expect.arrayContaining(['packages.description', 'maintainers.display_name']), + ) }) it('skips maintainers for a non-critical package but still links the repo', async () => { @@ -187,6 +197,7 @@ describe('persistPackagistPackageInfo', () => { expect(result).toEqual({ found: false, changedFields: [] }) expect(mockRepoGet).not.toHaveBeenCalled() expect(mockMaintainers).not.toHaveBeenCalled() + expect(mockAudit).not.toHaveBeenCalled() }) it('strips NUL bytes from the description before writing (Postgres rejects them)', async () => { diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index 83fa211af4..7ae36a9dd1 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -147,9 +147,10 @@ export async function ingestOnePackagistMetadata( } const stats = normalizePackagistStats(info.value.package) - const persistedInfo = await persistPackagistPackageInfo(qx, candidate.purl, stats) - // Audit phase 1 immediately: those writes are already committed - await logAuditFieldChanges(qx, WORKER, candidate.purl, persistedInfo.changedFields) + // persistPackagistPackageInfo audits its own writes atomically, inside the same + // transaction — phase 1 is committed-and-audited before the p2 fetch (which can + // throw) ever runs. + await persistPackagistPackageInfo(qx, candidate.purl, stats) // Phase 2: p2 endpoint const p2 = await fetchWithFastRetry( @@ -172,9 +173,10 @@ export async function ingestOnePackagistMetadata( } let lastModified: string | null = null - let phase2ChangedFields: string[] = [] if (!isP2NotModified(p2.value)) { const expanded = expandComposerMetadata(p2.value.minifiedVersions) + // persistPackagistMetadata audits its own writes atomically, inside the same + // transaction as the aggregate/version/dependency writes. const persistResult = await persistPackagistMetadata(qx, candidate.purl, expanded) if (persistResult.unresolvedDependencyTargets > 0) { log.debug( @@ -182,11 +184,9 @@ export async function ingestOnePackagistMetadata( 'packagist dependency targets not found in packages — edges skipped', ) } - phase2ChangedFields = persistResult.changedFields lastModified = p2.value.lastModified } - await logAuditFieldChanges(qx, WORKER, candidate.purl, phase2ChangedFields) await markPackagistMetadataScanned( qx, candidate.purl, @@ -219,13 +219,9 @@ export async function ingestOnePackagist30dWindow( return } - const changedFields = await persistPackagist30dWindow( - qx, - purl, - info.value.package.downloads?.monthly ?? null, - runDate, - ) - await logAuditFieldChanges(qx, WORKER, purl, changedFields) + // persistPackagist30dWindow audits its own write atomically, inside the same + // transaction as the insert-if-absent. + await persistPackagist30dWindow(qx, purl, info.value.package.downloads?.monthly ?? null, runDate) await markPackagist30dProcessed(qx, purl, { status: 'success', attempts: info.attempts }) } @@ -258,10 +254,15 @@ export async function ingestOnePackagistDailyDownload( const daily = info.value.package.downloads?.daily if (typeof daily === 'number') { - const changedFields = await insertDailyDownloads(qx, candidate.packageId, [ - { day: runDate, downloads: daily }, - ]) - await logAuditFieldChanges(qx, WORKER, candidate.purl, changedFields) + // Insert + audit share one transaction so a failed audit insert can never leave a + // committed row unaudited — a retry would hit ON CONFLICT DO NOTHING and report no + // changes, permanently losing the audit event otherwise. + await qx.tx(async (t) => { + const changedFields = await insertDailyDownloads(t, candidate.packageId, [ + { day: runDate, downloads: daily }, + ]) + await logAuditFieldChanges(t, WORKER, candidate.purl, changedFields) + }) } await markPackagistDailyProcessed(qx, candidate.purl, { status: 'success', diff --git a/services/apps/packages_worker/src/packagist/downloads.ts b/services/apps/packages_worker/src/packagist/downloads.ts index 1034302499..f8d609a218 100644 --- a/services/apps/packages_worker/src/packagist/downloads.ts +++ b/services/apps/packages_worker/src/packagist/downloads.ts @@ -1,6 +1,11 @@ -import { insertLast30dDownloadIfAbsent } from '@crowd/data-access-layer/src/packages' +import { + insertLast30dDownloadIfAbsent, + logAuditFieldChanges, +} from '@crowd/data-access-layer/src/packages' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +const WORKER = 'packagist' + // Compute the monthly window for observed rolling 30d downloads. // endDate = first of the run month; startDate = endDate minus 30 days. export function monthlyWindowFor(runDate: string): { startDate: string; endDate: string } { @@ -21,8 +26,8 @@ export function monthlyWindowFor(runDate: string): { startDate: string; endDate: // the value is the observation closest to the boundary. insertLast30dDownloadIfAbsent // does the presence check and the insert (+ mirror) atomically in one statement, so // concurrent runs for the same purl+month can't both pass a check-then-insert gap and -// have the later one clobber the earlier one's count. Returns the changed fields (like -// the sibling persist*/upsert* functions) — the caller audits them. +// have the later one clobber the earlier one's count. The audit record shares the same +// transaction, so a failed audit insert can never leave a committed window unaudited. export async function persistPackagist30dWindow( qx: QueryExecutor, purl: string, @@ -32,5 +37,16 @@ export async function persistPackagist30dWindow( if (monthly == null) return [] const { startDate, endDate } = monthlyWindowFor(runDate) - return insertLast30dDownloadIfAbsent(qx, purl, startDate, endDate, monthly, true) + return qx.tx(async (t) => { + const changedFields = await insertLast30dDownloadIfAbsent( + t, + purl, + startDate, + endDate, + monthly, + true, + ) + await logAuditFieldChanges(t, WORKER, purl, changedFields) + return changedFields + }) } diff --git a/services/apps/packages_worker/src/packagist/upsertMetadata.ts b/services/apps/packages_worker/src/packagist/upsertMetadata.ts index 87114b5dc9..bdd594c8f9 100644 --- a/services/apps/packages_worker/src/packagist/upsertMetadata.ts +++ b/services/apps/packages_worker/src/packagist/upsertMetadata.ts @@ -1,5 +1,6 @@ import { getPackagistPackageIdsByNames, + logAuditFieldChanges, reconcileVersionDependencies, updatePackagistVersionAggregates, upsertPackagistVersions, @@ -16,6 +17,8 @@ import { } from './normalize' import type { PackagistDependency, PackagistExpandedVersion } from './types' +const WORKER = 'packagist' + export async function persistPackagistMetadata( qx: QueryExecutor, purl: string, @@ -32,10 +35,11 @@ export async function persistPackagistMetadata( const changedFields: string[] = [] let unresolvedDependencyTargets = 0 - // One transaction for the whole p2 write path: the aggregates update, version - // upsert + is_latest cleanup, and dependency reconcile (delete stale + upsert - // current) are each multi-statement on their own — sharing one tx means a failure - // partway through can never leave versions/dependencies half-refreshed. + // One transaction for the whole p2 write path AND its audit record: the aggregates + // update, version upsert + is_latest cleanup, and dependency reconcile (delete stale + // + upsert current) are each multi-statement on their own — sharing one tx means a + // failure partway through (including a failed audit insert) can never leave + // versions/dependencies half-refreshed or their audit trail out of sync. await qx.tx(async (t) => { const agg = await updatePackagistVersionAggregates(t, purl, { versionsCount: versionRows.length, @@ -106,6 +110,8 @@ export async function persistPackagistMetadata( ) changedFields.push(...depChanges) } + + await logAuditFieldChanges(t, WORKER, purl, changedFields) }) return { found, changedFields, unresolvedDependencyTargets } diff --git a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts index 775dd0a70f..4639c05dd0 100644 --- a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts +++ b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts @@ -1,5 +1,6 @@ import { getOrCreateRepoByUrl, + logAuditFieldChanges, removeDeclaredPackageRepo, updatePackagistPackageStats, upsertPackageMaintainers, @@ -12,11 +13,15 @@ import { stripNullBytesDeep } from '../utils/stripNullBytesDeep' import type { NormalizedPackagistStats } from './types' +const WORKER = 'packagist' + // Dynamic-endpoint persistence: packages fields + repo link for ALL packages, // maintainers only for critical ones. Download rows are NOT written here — they -// belong to the dedicated downloads-30d/daily lanes. All writes share one -// transaction so a failure partway through can never leave the packages row, -// repo link, and maintainer set inconsistent with each other. +// belong to the dedicated downloads-30d/daily lanes. All writes AND the audit +// record share one transaction so a failure partway through — including a +// failed audit insert — can never leave the writes and their audit trail +// inconsistent with each other, and a retry can't lose an already-committed +// change's audit event. export async function persistPackagistPackageInfo( qx: QueryExecutor, purl: string, @@ -85,6 +90,8 @@ export async function persistPackagistPackageInfo( ) changedFields.push(...maintainerChanges) } + + await logAuditFieldChanges(t, WORKER, purl, changedFields) }) return { found, changedFields } From fde7e7e114a27ca22bf299c03ebf5e42dc12eddd Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 18:36:22 +0200 Subject: [PATCH 37/56] fix: allow double hyphens in Composer project names Signed-off-by: anilb --- .../packagist/__tests__/listPackages.test.ts | 19 +++++++++++++++++++ .../src/packagist/listPackages.ts | 13 +++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts b/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts index 0862097d66..5f792a5005 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts @@ -30,6 +30,16 @@ describe('parsePackagistPackageList', () => { ]) }) + it('accepts up to two consecutive hyphens in the project segment only', () => { + // Composer's own name spec allows `-{1,2}` as a separator in the project segment + // but only a single separator in the vendor segment. + const { entries, invalid } = parsePackagistPackageList({ + packageNames: ['vendor/my--package', 'vendor/my---package', 'my--vendor/package'], + }) + expect(entries.map((e) => e.purl)).toEqual(['pkg:composer/vendor/my--package']) + expect(invalid).toBe(2) + }) + it('skips and counts invalid names', () => { const { entries, invalid } = parsePackagistPackageList({ packageNames: [ @@ -58,6 +68,15 @@ describe('parsePackagistPackageList', () => { expect(invalid).toBe(1) }) + it('rejects a long run of hyphens without catastrophic backtracking (project -{1,2} alternative)', () => { + const pathological = 'a' + '-'.repeat(50) + '!/name' + const start = Date.now() + const { entries, invalid } = parsePackagistPackageList({ packageNames: [pathological] }) + expect(Date.now() - start).toBeLessThan(200) + expect(entries).toEqual([]) + expect(invalid).toBe(1) + }) + it('lowercases and deduplicates names without counting duplicates as invalid', () => { const { entries, invalid } = parsePackagistPackageList({ packageNames: ['Monolog/Monolog', 'monolog/monolog'], diff --git a/services/apps/packages_worker/src/packagist/listPackages.ts b/services/apps/packages_worker/src/packagist/listPackages.ts index 4b3e099af9..8e561ed9be 100644 --- a/services/apps/packages_worker/src/packagist/listPackages.ts +++ b/services/apps/packages_worker/src/packagist/listPackages.ts @@ -7,7 +7,12 @@ const PACKAGIST_LIST = 'https://packagist.org/packages/list.json' // Non-backtracking form: a mandatory separator per repetition (vs. an optional one) // removes the ambiguity that let the old `([_.-]?[a-z0-9]+)*` pattern backtrack // exponentially on a long run of the same character class (CodeQL js/redos). -const COMPOSER_NAME_REGEX = /^[a-z0-9]+(?:[_.-][a-z0-9]+)*$/ +// Vendor and project segments have different rules per Composer's own name spec: the +// vendor allows only a single separator between alnum runs, but the project segment +// additionally allows up to two consecutive hyphens (e.g. `vendor/my--package` is a +// valid, real Composer name) — verified against Composer's ArrayLoader name pattern. +const COMPOSER_VENDOR_REGEX = /^[a-z0-9]+(?:[_.-][a-z0-9]+)*$/ +const COMPOSER_PROJECT_REGEX = /^[a-z0-9]+(?:(?:[_.]|-{1,2})[a-z0-9]+)*$/ export interface PackagistListEntry { vendor: string @@ -41,13 +46,13 @@ export function parsePackagistPackageList(json: unknown): { const lowercased = item.toLowerCase() const parts = lowercased.split('/') - // Validate: exactly one slash, each side non-empty and matches composer name pattern + // Validate: exactly one slash, each side non-empty and matches its composer name pattern if ( parts.length !== 2 || !parts[0] || !parts[1] || - !COMPOSER_NAME_REGEX.test(parts[0]) || - !COMPOSER_NAME_REGEX.test(parts[1]) + !COMPOSER_VENDOR_REGEX.test(parts[0]) || + !COMPOSER_PROJECT_REGEX.test(parts[1]) ) { invalid++ continue From 5da5ababe7e8d1711c503146682a0d0069f490d8 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 18:39:55 +0200 Subject: [PATCH 38/56] fix: drain packagist response bodies on error paths Signed-off-by: anilb --- .../packagist/__tests__/fetchPackage.test.ts | 37 +++++++++++------ .../src/packagist/fetchPackage.ts | 40 ++++++++++++++++--- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts b/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts index c0dd70d4df..256092c5dd 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts @@ -2,12 +2,13 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { buildPackagistUserAgent, fetchPackagistP2, fetchPackagistStats } from '../fetchPackage' -// Minimal Response stand-in for the global fetch mock. +// Minimal Response stand-in for the global fetch mock. `body.cancel` is a spy so +// error-path tests can assert the response body is drained before returning. function fakeResponse( status: number, body?: unknown, opts: { jsonThrows?: boolean; lastModified?: string } = {}, -): Response { +): Response & { body: { cancel: ReturnType } } { return { status, ok: status >= 200 && status < 300, @@ -19,7 +20,8 @@ function fakeResponse( if (opts.jsonThrows) throw new Error('bad json') return body }, - } as unknown as Response + body: { cancel: vi.fn() }, + } as unknown as Response & { body: { cancel: ReturnType } } } const validStats = { package: { name: 'monolog/monolog', downloads: { monthly: 5 } } } @@ -65,28 +67,35 @@ describe('fetchPackagistStats', () => { expect(headers['User-Agent']).toMatch(/mailto=/) }) - it('maps 404 → NOT_FOUND', async () => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(404))) + it('maps 404 → NOT_FOUND and drains the response body', async () => { + const res = fakeResponse(404) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(res)) expect(await fetchPackagistStats('gone/gone')).toMatchObject({ kind: 'NOT_FOUND', statusCode: 404, }) + // an undrained body can pin the socket instead of returning it to the shared pool + expect(res.body.cancel).toHaveBeenCalled() }) - it('maps 429 → RATE_LIMIT', async () => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(429))) + it('maps 429 → RATE_LIMIT and drains the response body', async () => { + const res = fakeResponse(429) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(res)) expect(await fetchPackagistStats('busy/busy')).toMatchObject({ kind: 'RATE_LIMIT', statusCode: 429, }) + expect(res.body.cancel).toHaveBeenCalled() }) - it('maps other non-ok statuses (5xx) → TRANSIENT with the status code', async () => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(503))) + it('maps other non-ok statuses (5xx) → TRANSIENT with the status code and drains the body', async () => { + const res = fakeResponse(503) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(res)) expect(await fetchPackagistStats('flaky/flaky')).toMatchObject({ kind: 'TRANSIENT', statusCode: 503, }) + expect(res.body.cancel).toHaveBeenCalled() }) it('maps a network rejection → TRANSIENT without a status code', async () => { @@ -170,12 +179,16 @@ describe('fetchPackagistP2', () => { }) }) - it('maps 404 → NOT_FOUND and 429 → RATE_LIMIT', async () => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(404))) + it('maps 404 → NOT_FOUND and 429 → RATE_LIMIT, draining the body each time', async () => { + const notFound = fakeResponse(404) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(notFound)) expect(await fetchPackagistP2('gone/gone', null)).toMatchObject({ kind: 'NOT_FOUND' }) + expect(notFound.body.cancel).toHaveBeenCalled() - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(429))) + const rateLimited = fakeResponse(429) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(rateLimited)) expect(await fetchPackagistP2('busy/busy', null)).toMatchObject({ kind: 'RATE_LIMIT' }) + expect(rateLimited.body.cancel).toHaveBeenCalled() }) it('maps a payload missing the package key → MALFORMED', async () => { diff --git a/services/apps/packages_worker/src/packagist/fetchPackage.ts b/services/apps/packages_worker/src/packagist/fetchPackage.ts index 266c11d454..03cf3d6f1e 100644 --- a/services/apps/packages_worker/src/packagist/fetchPackage.ts +++ b/services/apps/packages_worker/src/packagist/fetchPackage.ts @@ -33,6 +33,18 @@ export function describeFetchFailure(err: unknown): string { return detail ? `${String(err)} (cause: ${detail})` : String(err) } +// An error-path response whose body is never read can pin its socket instead of +// returning it to packagistDispatcher's pool (capped at 10 connections per origin) — +// canceling it releases the connection for reuse. Best-effort: a failure here must +// never mask the real error already being returned. +async function discardBody(res: Response): Promise { + try { + await res.body?.cancel() + } catch { + // ignore + } +} + export async function fetchPackagistStats(name: string): Promise { const url = `https://packagist.org/packages/${name}.json` const abort = new AbortController() @@ -55,10 +67,18 @@ export async function fetchPackagistStats(name: string): Promise Date: Fri, 17 Jul 2026 18:44:40 +0200 Subject: [PATCH 39/56] test: verify packagist 30d insert race against real Postgres Signed-off-by: anilb --- .../downloadsLast30d.integration.test.ts | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 services/libs/data-access-layer/src/packages/downloadsLast30d.integration.test.ts diff --git a/services/libs/data-access-layer/src/packages/downloadsLast30d.integration.test.ts b/services/libs/data-access-layer/src/packages/downloadsLast30d.integration.test.ts new file mode 100644 index 0000000000..ceac6c2190 --- /dev/null +++ b/services/libs/data-access-layer/src/packages/downloadsLast30d.integration.test.ts @@ -0,0 +1,146 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { getDbConnection } from '@crowd/database' + +import type { QueryExecutor } from '../queryExecutor' +import { pgpQx } from '../queryExecutor' + +import { insertLast30dDownloadIfAbsent } from './downloadsLast30d' + +// Integration test: hits the running packages-db. Skipped automatically when any of +// the DB env vars are missing so unit-test runs in CI stay green. +const HAVE_DB = + !!process.env.CROWD_PACKAGES_DB_WRITE_HOST && + !!process.env.CROWD_PACKAGES_DB_PORT && + !!process.env.CROWD_PACKAGES_DB_USERNAME && + !!process.env.CROWD_PACKAGES_DB_DATABASE && + !!process.env.CROWD_PACKAGES_DB_PASSWORD + +const FIXTURE_TAG = 'akrites-downloads-last-30d-fixture' + +// insertLast30dDownloadIfAbsent is packagist's "first observation wins" guard for the +// monthly rolling window: the presence check, insert, and packages mirror all happen +// in one statement (ON CONFLICT DO NOTHING) so a genuine race between two callers for +// the same purl+month can never let a later write silently overwrite an earlier one. +describe.skipIf(!HAVE_DB)('insertLast30dDownloadIfAbsent — real packages-db', () => { + let qx: QueryExecutor + + async function cleanupFixtures(): Promise { + await qx.result( + `DELETE FROM downloads_last_30d WHERE purl IN ( + SELECT purl FROM packages WHERE ingestion_source = $(tag))`, + { tag: FIXTURE_TAG }, + ) + await qx.result(`DELETE FROM packages WHERE ingestion_source = $(tag)`, { tag: FIXTURE_TAG }) + } + + async function makePackage(purl: string): Promise { + await qx.result( + `INSERT INTO packages (purl, ecosystem, namespace, name, registry_url, status, ingestion_source) + VALUES ($(purl), 'packagist', 'fixture', $(purl), 'https://example.test', 'active', $(tag))`, + { purl, tag: FIXTURE_TAG }, + ) + } + + beforeAll(async () => { + const conn = await getDbConnection({ + host: process.env.CROWD_PACKAGES_DB_WRITE_HOST ?? '', + port: parseInt(process.env.CROWD_PACKAGES_DB_PORT ?? '0', 10), + database: process.env.CROWD_PACKAGES_DB_DATABASE ?? '', + user: process.env.CROWD_PACKAGES_DB_USERNAME ?? '', + password: process.env.CROWD_PACKAGES_DB_PASSWORD ?? '', + }) + qx = pgpQx(conn) + await cleanupFixtures() + }, 30_000) + + afterAll(async () => { + if (qx) await cleanupFixtures() + }) + + it('inserts the window and mirrors it to packages on the first call', async () => { + const purl = `pkg:composer/${FIXTURE_TAG}/first-call` + await makePackage(purl) + + const changed = await insertLast30dDownloadIfAbsent( + qx, + purl, + '2026-06-01', + '2026-07-01', + 100, + true, + ) + + expect(changed).toEqual( + expect.arrayContaining(['downloads_last_30d.start_date', 'downloads_last_30d.count']), + ) + expect(changed).toContain('packages.downloads_last_30d') + + const row = await qx.selectOne(`SELECT count FROM downloads_last_30d WHERE purl = $(purl)`, { + purl, + }) + expect(Number(row.count)).toBe(100) + const pkg = await qx.selectOne(`SELECT downloads_last_30d FROM packages WHERE purl = $(purl)`, { + purl, + }) + expect(Number(pkg.downloads_last_30d)).toBe(100) + }) + + it('does not overwrite an existing window — later call with a different count is a no-op', async () => { + const purl = `pkg:composer/${FIXTURE_TAG}/second-call-loses` + await makePackage(purl) + + const first = await insertLast30dDownloadIfAbsent( + qx, + purl, + '2026-06-01', + '2026-07-01', + 100, + true, + ) + expect(first.length).toBeGreaterThan(0) + + const second = await insertLast30dDownloadIfAbsent( + qx, + purl, + '2026-06-01', + '2026-07-01', + 999, + true, + ) + expect(second).toEqual([]) + + const row = await qx.selectOne(`SELECT count FROM downloads_last_30d WHERE purl = $(purl)`, { + purl, + }) + expect(Number(row.count)).toBe(100) + const pkg = await qx.selectOne(`SELECT downloads_last_30d FROM packages WHERE purl = $(purl)`, { + purl, + }) + expect(Number(pkg.downloads_last_30d)).toBe(100) + }) + + it('resolves a genuine race between concurrent callers to exactly one consistent winner', async () => { + const purl = `pkg:composer/${FIXTURE_TAG}/concurrent-race` + await makePackage(purl) + + const [a, b] = await Promise.all([ + insertLast30dDownloadIfAbsent(qx, purl, '2026-06-01', '2026-07-01', 111, true), + insertLast30dDownloadIfAbsent(qx, purl, '2026-06-01', '2026-07-01', 222, true), + ]) + + // exactly one of the two concurrent callers observes itself as the writer + const winners = [a, b].filter((c) => c.length > 0) + expect(winners).toHaveLength(1) + + const row = await qx.selectOne(`SELECT count FROM downloads_last_30d WHERE purl = $(purl)`, { + purl, + }) + const pkg = await qx.selectOne(`SELECT downloads_last_30d FROM packages WHERE purl = $(purl)`, { + purl, + }) + // whichever value won, the window row and its packages mirror must agree + expect(Number(pkg.downloads_last_30d)).toBe(Number(row.count)) + expect([111, 222]).toContain(Number(row.count)) + }) +}) From fce90782d02d9a46857457b70e4a0a4f3cadedf2 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 18:46:25 +0200 Subject: [PATCH 40/56] docs: fix stale ADR-0006 references, should be ADR-0009 Signed-off-by: anilb --- services/apps/packages_worker/src/packagist/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index 8c31c6a442..d26ed3b6ce 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -47,7 +47,7 @@ Shared design notes: with the per-item give-up so one bad package can never stall a drain). - **Tier scoping.** Discovery, package info, repo links, versions, and dependencies cover **all** packages (deliberate divergence from pypi — see - ADR-0006). Only `downloads_daily` and maintainers are + ADR-0009). Only `downloads_daily` and maintainers are **critical-slice-only**, matching npm/pypi. --- @@ -105,7 +105,7 @@ touching p2. `is_prerelease`, `is_latest`, `licenses`); dev branches skipped. - `package_dependencies` — direct edges only (`require` → `'direct'`, `require-dev` → `'dev'`), declared constraints verbatim, platform packages - excluded, `depends_on_version_id` NULL (resolved at query time — ADR-0006). + excluded, `depends_on_version_id` NULL (resolved at query time — ADR-0009). - `packages` aggregates — `versions_count`, `latest_version`, `first_release_at`, `latest_release_at`, `licenses`, `homepage` — all written non-destructively (COALESCE / keep-on-zero). @@ -169,7 +169,7 @@ last. State: `daily_downloads_last_run_at` + `daily_downloads_run_result`. - `transitive_dependent_count`, `dependent_repos_count` — not computable from the registry; needs a reverse-closure over our stored direct edges - (future work, see ADR-0006 risks). + (future work, see ADR-0009 risks). - Advisories — the OSV worker owns security data platform-wide. - `is_critical` / `criticality_score` / ranking columns — the shared criticality worker; this worker only *reads* `is_critical` for scoping. From 6fe45a45666b092df27c1d6f8311a0b02dd5d200 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 20:17:14 +0200 Subject: [PATCH 41/56] style: fix pre-existing prettier drift in packagist README Signed-off-by: anilb --- .../packages_worker/src/packagist/README.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index d26ed3b6ce..558ae7b11b 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -3,7 +3,7 @@ The Packagist worker keeps the PHP/Composer slice of the packages database fresh by crawling **packagist.org directly** — deps.dev has no Packagist coverage, so unlike npm/maven/pypi there is no BigQuery universe to import from; the registry -crawl *is* the universe source. It runs **four Temporal workflows** on the +crawl _is_ the universe source. It runs **four Temporal workflows** on the `packagist-worker` task queue: three cron schedules registered at worker boot (`src/bin/packagist-worker.ts`), plus the metadata drain, which the seed chains as a child workflow on completion. @@ -13,10 +13,10 @@ Identity: ecosystem `packagist`, purls `pkg:composer/{vendor}/{name}` Shared design notes: -- **Two registry endpoints.** The *dynamic* endpoint +- **Two registry endpoints.** The _dynamic_ endpoint (`packagist.org/packages/{vendor}/{name}.json`, server-cached ~12h) carries package info and live counters: downloads, dependents, description, repo URL, - abandoned flag, maintainers. The *static* p2 endpoint + abandoned flag, maintainers. The _static_ p2 endpoint (`repo.packagist.org/p2/{vendor}/{name}.json`, CDN-served, the same file Composer clients resolve) carries the version manifests: every tagged release with its `require`/`require-dev`, licenses, homepage. @@ -158,7 +158,7 @@ the registry's rolling daily figure. **Populates:** one `downloads_daily` row per package per day (`package_id`, run date, count), with the run date pinned once from the run's `cutoff` so every row from one run shares the same label even if the drain runs long. -That only fixes the *label*, though — Packagist computes `daily` live at +That only fixes the _label_, though — Packagist computes `daily` live at fetch time, so for a long-running drain the underlying value can drift forward from what the label implies for whichever packages are processed last. State: `daily_downloads_last_run_at` + `daily_downloads_run_result`. @@ -172,7 +172,7 @@ last. State: `daily_downloads_last_run_at` + `daily_downloads_run_result`. (future work, see ADR-0009 risks). - Advisories — the OSV worker owns security data platform-wide. - `is_critical` / `criticality_score` / ranking columns — the shared - criticality worker; this worker only *reads* `is_critical` for scoping. + criticality worker; this worker only _reads_ `is_critical` for scoping. - Repo stars/forks/scorecard — the shared github-repos-enricher, fed by the `package_repos` links this worker creates. - `keywords`, `versions.is_yanked`, `depends_on_version_id` — see above. @@ -181,13 +181,13 @@ last. State: `daily_downloads_last_run_at` + `daily_downloads_run_result`. All optional; defaults in `activities.ts` / `fetchPackage.ts`: -| Env var | Default | Meaning | -|---|---|---| -| `CROWD_PACKAGES_PACKAGIST_MAILTO` | `oss-packages@linuxfoundation.org` | Contact in the crawl User-Agent | -| `CROWD_PACKAGES_PACKAGIST_STATS_CONCURRENCY` | 10 (hard cap 10) | Parallel package ingests across all lanes — every lane starts with a dynamic-endpoint fetch, so the stricter 10-limit bounds everything | -| `CROWD_PACKAGES_PACKAGIST_METADATA_REFRESH_DAYS` | 7 | Metadata refresh window | -| `CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL` | `false` | `true` narrows metadata to the critical slice | -| `CROWD_PACKAGES_PACKAGIST_STOP_AFTER_FIRST_PAGE` | `false` | Debug: one drain page, no continueAsNew | +| Env var | Default | Meaning | +| ------------------------------------------------ | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| `CROWD_PACKAGES_PACKAGIST_MAILTO` | `oss-packages@linuxfoundation.org` | Contact in the crawl User-Agent | +| `CROWD_PACKAGES_PACKAGIST_STATS_CONCURRENCY` | 10 (hard cap 10) | Parallel package ingests across all lanes — every lane starts with a dynamic-endpoint fetch, so the stricter 10-limit bounds everything | +| `CROWD_PACKAGES_PACKAGIST_METADATA_REFRESH_DAYS` | 7 | Metadata refresh window | +| `CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL` | `false` | `true` narrows metadata to the critical slice | +| `CROWD_PACKAGES_PACKAGIST_STOP_AFTER_FIRST_PAGE` | `false` | Debug: one drain page, no continueAsNew | ## Running it From 4c55efeda4d1d1e4ad6427a6bb60d2fef2da91c0 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 22:41:15 +0200 Subject: [PATCH 42/56] fix: renumber packagist migration version Signed-off-by: anilb --- ...__packagist_worker.sql => V1784314023__packagist_worker.sql} | 0 services/apps/packages_worker/src/packagist/README.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename backend/src/osspckgs/migrations/{V1783382400__packagist_worker.sql => V1784314023__packagist_worker.sql} (100%) diff --git a/backend/src/osspckgs/migrations/V1783382400__packagist_worker.sql b/backend/src/osspckgs/migrations/V1784314023__packagist_worker.sql similarity index 100% rename from backend/src/osspckgs/migrations/V1783382400__packagist_worker.sql rename to backend/src/osspckgs/migrations/V1784314023__packagist_worker.sql diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index 558ae7b11b..7edb6157b2 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -207,5 +207,5 @@ Note: triggering `seed` also chain-starts the full `metadata` drain (set `CROWD_PACKAGES_PACKAGIST_STOP_AFTER_FIRST_PAGE=true` locally to bound it). Local smoke order: seed → metadata → (rank) → downloads lanes for critical-scoped writes. State lives in `packagist_package_state` -(migration `V1783382400__packagist_worker.sql`); design decisions in +(migration `V1784314023__packagist_worker.sql`); design decisions in `docs/adr/0009-packagist-worker-design-decisions.md`. From 6c0c2147ffc2889353104eae59f5d08865af5e02 Mon Sep 17 00:00:00 2001 From: anilb Date: Fri, 17 Jul 2026 22:41:25 +0200 Subject: [PATCH 43/56] fix: validate packagist response shapes Signed-off-by: anilb --- .../packagist/__tests__/fetchPackage.test.ts | 50 +++++++++++++++++++ .../src/packagist/__tests__/normalize.test.ts | 12 +++++ .../src/packagist/fetchPackage.ts | 40 ++++++++++++++- .../src/packagist/normalize.ts | 8 ++- 4 files changed, 106 insertions(+), 4 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts b/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts index 256092c5dd..79783c20df 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts @@ -118,6 +118,21 @@ describe('fetchPackagistStats', () => { expect(await fetchPackagistStats('monolog/monolog')).toMatchObject({ kind: 'MALFORMED' }) }) + // Regression: a technically-valid-JSON body with the wrong runtime type for a field + // normalizePackagistStats consumes unconditionally must be classified MALFORMED, not + // thrown past the guard (where it would be misread as a transient failure and + // retried forever on the same deterministic input). + it.each([ + ['description is a number', { package: { name: 'a/b', description: 123 } }], + ['repository is an object', { package: { name: 'a/b', repository: {} } }], + ['maintainers is not an array', { package: { name: 'a/b', maintainers: {} } }], + ])('maps a wrong-typed field (%s) → MALFORMED, not a throw', async (_desc, body) => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(200, body))) + await expect(fetchPackagistStats('monolog/monolog')).resolves.toMatchObject({ + kind: 'MALFORMED', + }) + }) + it('maps a body read aborted by the 30s timeout → TRANSIENT (not MALFORMED)', async () => { vi.useFakeTimers() vi.stubGlobal( @@ -198,4 +213,39 @@ describe('fetchPackagistP2', () => { ) expect(await fetchPackagistP2('monolog/monolog', null)).toMatchObject({ kind: 'MALFORMED' }) }) + + // Regression: a malformed element in the version array must be classified + // MALFORMED, not thrown past the guard — expandComposerMetadata's Object.entries() + // throws on a null/non-object element, and version/version_normalized/license reach + // .startsWith()/.split()/.endsWith() and the SQL text[] write unconditionally. + it.each([ + ['an element is null', [null]], + ['version is missing', [{ time: '2024-01-01' }]], + ['version is a number', [{ version: 123 }]], + ['version_normalized is a number', [{ version: '1.0.0', version_normalized: 42 }]], + ['license is a scalar string, not an array', [{ version: '1.0.0', license: 'MIT' }]], + ['homepage is an object', [{ version: '1.0.0', homepage: {} }]], + ['time is a number', [{ version: '1.0.0', time: 20240101 }]], + ])('maps a malformed version entry (%s) → MALFORMED, not a throw', async (_desc, versions) => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(fakeResponse(200, { packages: { 'monolog/monolog': versions } })), + ) + await expect(fetchPackagistP2('monolog/monolog', null)).resolves.toMatchObject({ + kind: 'MALFORMED', + }) + }) + + it('accepts a version entry using the __unset diff sentinel on optional fields', async () => { + const versions = [ + { version: '1.0.0', license: ['MIT'] }, + { version: '2.0.0', license: '__unset' }, + ] + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(fakeResponse(200, { packages: { 'monolog/monolog': versions } })), + ) + const result = await fetchPackagistP2('monolog/monolog', null) + expect(result).not.toMatchObject({ kind: 'MALFORMED' }) + }) }) diff --git a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts index 0b17389691..2974c306f2 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts @@ -55,6 +55,18 @@ describe('normalizePackagistStats', () => { expect(normalizePackagistStats(full).status).toBe('active') }) + it('drops a null/non-object maintainer entry instead of throwing', () => { + // isPackagistStatsJson only guarantees maintainers is an array, not that every + // element is a well-formed object — a rogue null must not crash `m.name` access. + const stats = normalizePackagistStats({ + name: 'a/b', + maintainers: [null, 'not-an-object', { name: 'seldaek' }] as never, + }) + expect(stats.maintainers).toEqual([ + { username: 'seldaek', displayName: null, email: null, role: 'maintainer' }, + ]) + }) + it('tolerates missing optional fields with nulls and empty lists', () => { const stats = normalizePackagistStats({ name: 'a/b' }) expect(stats).toEqual({ diff --git a/services/apps/packages_worker/src/packagist/fetchPackage.ts b/services/apps/packages_worker/src/packagist/fetchPackage.ts index 03cf3d6f1e..12dd3a5159 100644 --- a/services/apps/packages_worker/src/packagist/fetchPackage.ts +++ b/services/apps/packages_worker/src/packagist/fetchPackage.ts @@ -171,11 +171,47 @@ export async function fetchPackagistP2( } } +// Validates every field normalizePackagistStats consumes unconditionally +// (blankToNull's .trim(), the maintainers .filter()) — a wrong runtime type here must +// surface as MALFORMED, not throw past the fast-retry/give-up path and get treated as +// a transient failure that Temporal retries forever on the same deterministic input. function isPackagistStatsJson(v: unknown): v is PackagistStatsJson { if (typeof v !== 'object' || v === null || !('package' in v)) return false const pkg = (v as { package: unknown }).package + if (typeof pkg !== 'object' || pkg === null) return false + const p = pkg as Record + if (typeof p.name !== 'string') return false + if (p.description != null && typeof p.description !== 'string') return false + if (p.repository != null && typeof p.repository !== 'string') return false + if (p.maintainers != null && !Array.isArray(p.maintainers)) return false + return true +} + +// '__unset' is the p2 minified-diff sentinel for "field removed since the previous +// entry" — a legitimate value for any optional field, not just an absent key. +function isValidVersionField(value: unknown, isValidType: (v: unknown) => boolean): boolean { + return value === undefined || value === '__unset' || isValidType(value) +} + +// Validates every field normalize.ts consumes unconditionally on an expanded version: +// `version`/`version_normalized` via .startsWith()/.split()/.endsWith() +// (isPackagistDevVersion/isPackagistPrerelease), `homepage` via blankToNull's .trim(), +// `license` as the text[] written to versions.licenses, `time` as the timestamptz +// written to versions.published_at. `require`/`require-dev` are already guarded at +// their own call site (extractVersionDependencies checks typeof before Object.entries) +// so aren't re-validated here. +function isValidMinifiedVersion(v: unknown): boolean { + if (typeof v !== 'object' || v === null) return false + const entry = v as Record return ( - typeof pkg === 'object' && pkg !== null && typeof (pkg as { name?: unknown }).name === 'string' + typeof entry.version === 'string' && + isValidVersionField(entry.version_normalized, (x) => typeof x === 'string') && + isValidVersionField(entry.homepage, (x) => typeof x === 'string') && + isValidVersionField(entry.time, (x) => typeof x === 'string') && + isValidVersionField( + entry.license, + (x) => Array.isArray(x) && x.every((l) => typeof l === 'string'), + ) ) } @@ -184,5 +220,5 @@ function isP2Response(v: unknown, name: string): v is { packages: Record)[name] - return Array.isArray(entries) + return Array.isArray(entries) && entries.every(isValidMinifiedVersion) } diff --git a/services/apps/packages_worker/src/packagist/normalize.ts b/services/apps/packages_worker/src/packagist/normalize.ts index a21c73258a..1dee0af392 100644 --- a/services/apps/packages_worker/src/packagist/normalize.ts +++ b/services/apps/packages_worker/src/packagist/normalize.ts @@ -34,9 +34,13 @@ export function normalizePackagistStats(pkg: PackagistPackageInfo): NormalizedPa const dependents = typeof pkg.dependents === 'number' ? pkg.dependents : null - // Extract maintainers with non-empty names + // Extract maintainers with non-empty names. isPackagistStatsJson only guarantees + // maintainers is an array — a rogue null/non-object element would throw on `m.name`. const maintainers = (pkg.maintainers ?? []) - .filter((m): m is { name: string } => typeof m.name === 'string' && !!m.name) + .filter( + (m): m is { name: string } => + typeof m === 'object' && m !== null && typeof m.name === 'string' && !!m.name, + ) .map((m) => ({ username: m.name, displayName: null, From e90a2b2537397f53bc89ec835de485d520051643 Mon Sep 17 00:00:00 2001 From: anilb Date: Sat, 18 Jul 2026 16:08:51 +0200 Subject: [PATCH 44/56] fix: stop metadata lane overwriting 30d download mirror Signed-off-by: anilb --- .../src/packagist/__tests__/dueSelection.test.ts | 2 -- .../src/packagist/__tests__/normalize.test.ts | 6 +++--- .../packagist/__tests__/persistPackageInfo.test.ts | 5 +++-- .../apps/packages_worker/src/packagist/normalize.ts | 6 +++--- services/apps/packages_worker/src/packagist/types.ts | 1 - .../src/packagist/upsertPackageInfo.ts | 1 - .../libs/data-access-layer/src/packages/packages.ts | 11 ++++++----- 7 files changed, 15 insertions(+), 17 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts index 3bd1c2ea22..84999d06e8 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts @@ -207,7 +207,6 @@ describe('updatePackagistPackageStats', () => { declaredRepositoryUrl: null, repositoryUrl: null, status: 'active', - downloadsLast30d: 1, totalDownloads: 2, dependentCount: 3, }), @@ -229,7 +228,6 @@ describe('updatePackagistPackageStats', () => { declaredRepositoryUrl: 'https://github.com/a/x', repositoryUrl: 'https://github.com/a/x', status: 'active', - downloadsLast30d: 1, totalDownloads: 2, dependentCount: 3, }) diff --git a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts index 2974c306f2..ce835c0b84 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts @@ -33,7 +33,7 @@ describe('normalizePackagistStats', () => { downloads: { total: 500_000_000, monthly: 10_000_000, daily: 350_000 }, } - it('extracts description, repository, dependents and the three download counters', () => { + it('extracts description, repository, dependents and total downloads', () => { const stats = normalizePackagistStats(full) expect(stats).toEqual({ name: 'monolog/monolog', @@ -42,7 +42,8 @@ describe('normalizePackagistStats', () => { status: 'active', dependents: 5423, downloadsTotal: 500_000_000, - downloadsMonthly: 10_000_000, + // downloads.monthly is deliberately not extracted — packages.downloads_last_30d + // belongs exclusively to the dedicated downloads-30d lane // nameless maintainer entries are dropped maintainers: [{ username: 'seldaek', displayName: null, email: null, role: 'maintainer' }], }) @@ -76,7 +77,6 @@ describe('normalizePackagistStats', () => { status: 'active', dependents: null, downloadsTotal: null, - downloadsMonthly: null, maintainers: [], }) }) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts index ac58470779..77eb4d6e58 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts @@ -41,7 +41,6 @@ const stats: NormalizedPackagistStats = { status: 'active', dependents: 42, downloadsTotal: 1000, - downloadsMonthly: 300, maintainers: [{ username: 'seldaek', displayName: null, email: null, role: 'maintainer' }], } @@ -70,11 +69,13 @@ describe('persistPackagistPackageInfo', () => { purl: PURL, description: 'logs', status: 'active', - downloadsLast30d: 300, totalDownloads: 1000, dependentCount: 42, }), ) + // packages.downloads_last_30d belongs exclusively to the dedicated downloads-30d + // lane's boundary-anchored snapshot — the metadata lane must never touch it + expect(mockUpdate.mock.calls[0][1]).not.toHaveProperty('downloadsLast30d') // canonicalized (lowercased) url + coarse host, linked with the manifest-declared convention expect(mockRepoGet).toHaveBeenCalledWith(qx, 'https://github.com/seldaek/monolog', 'github') expect(mockRepoLink).toHaveBeenCalledWith(qx, '7', '55', 'declared', 0.8) diff --git a/services/apps/packages_worker/src/packagist/normalize.ts b/services/apps/packages_worker/src/packagist/normalize.ts index 1dee0af392..cd9bb937a7 100644 --- a/services/apps/packages_worker/src/packagist/normalize.ts +++ b/services/apps/packages_worker/src/packagist/normalize.ts @@ -28,9 +28,10 @@ export function normalizePackagistStats(pkg: PackagistPackageInfo): NormalizedPa status = 'deprecated' } - // Extract downloads, defaulting to null if not a number + // Extract downloads, defaulting to null if not a number. downloads.monthly is + // deliberately not extracted here — packages.downloads_last_30d belongs exclusively + // to the dedicated downloads-30d lane's boundary-anchored snapshot. const downloadsTotal = typeof pkg.downloads?.total === 'number' ? pkg.downloads.total : null - const downloadsMonthly = typeof pkg.downloads?.monthly === 'number' ? pkg.downloads.monthly : null const dependents = typeof pkg.dependents === 'number' ? pkg.dependents : null @@ -55,7 +56,6 @@ export function normalizePackagistStats(pkg: PackagistPackageInfo): NormalizedPa status, dependents, downloadsTotal, - downloadsMonthly, maintainers, } } diff --git a/services/apps/packages_worker/src/packagist/types.ts b/services/apps/packages_worker/src/packagist/types.ts index d514044e98..180a3bbb67 100644 --- a/services/apps/packages_worker/src/packagist/types.ts +++ b/services/apps/packages_worker/src/packagist/types.ts @@ -66,7 +66,6 @@ export interface NormalizedPackagistStats { status: 'active' | 'deprecated' dependents: number | null downloadsTotal: number | null - downloadsMonthly: number | null maintainers: PackageMaintainerInput[] } diff --git a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts index 4639c05dd0..860623fb3d 100644 --- a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts +++ b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts @@ -50,7 +50,6 @@ export async function persistPackagistPackageInfo( declaredRepositoryUrl: stats.repositoryUrl, repositoryUrl: trustedRepo?.url ?? null, status: stats.status, - downloadsLast30d: stats.downloadsMonthly, totalDownloads: stats.downloadsTotal, dependentCount: stats.dependents, }) diff --git a/services/libs/data-access-layer/src/packages/packages.ts b/services/libs/data-access-layer/src/packages/packages.ts index 164305a41a..ea987f9b51 100644 --- a/services/libs/data-access-layer/src/packages/packages.ts +++ b/services/libs/data-access-layer/src/packages/packages.ts @@ -259,11 +259,14 @@ export interface PackagistStatsUpdateInput { declaredRepositoryUrl: string | null repositoryUrl: string | null status: string - downloadsLast30d: number | null totalDownloads: number | null dependentCount: number | null } +// downloads_last_30d is NOT written here — it belongs exclusively to the dedicated +// downloads-30d lane's boundary-anchored, insert-if-absent snapshot. The metadata +// lane's live rolling `downloads.monthly` value is a different, unanchored reading; +// writing it here would silently overwrite that snapshot every weekly refresh. export async function updatePackagistPackageStats( qx: QueryExecutor, input: PackagistStatsUpdateInput, @@ -272,7 +275,7 @@ export async function updatePackagistPackageStats( await qx.selectOneOrNone( `WITH old AS ( SELECT description, declared_repository_url, repository_url, status, - downloads_last_30d, total_downloads, dependent_count, ingestion_source + total_downloads, dependent_count, ingestion_source FROM packages WHERE purl = $(purl) AND ecosystem = 'packagist' ), ins AS ( @@ -281,13 +284,12 @@ export async function updatePackagistPackageStats( declared_repository_url = $(declaredRepositoryUrl), repository_url = $(repositoryUrl), status = $(status), - downloads_last_30d = COALESCE($(downloadsLast30d), downloads_last_30d), total_downloads = COALESCE($(totalDownloads), total_downloads), dependent_count = COALESCE($(dependentCount), dependent_count), last_synced_at = NOW() WHERE purl = $(purl) AND ecosystem = 'packagist' RETURNING id, is_critical, description, declared_repository_url, repository_url, status, - downloads_last_30d, total_downloads, dependent_count, ingestion_source + total_downloads, dependent_count, ingestion_source ) SELECT ins.id::text AS id, ins.is_critical, array_remove(ARRAY[ @@ -295,7 +297,6 @@ export async function updatePackagistPackageStats( CASE WHEN o.declared_repository_url IS DISTINCT FROM ins.declared_repository_url THEN 'packages.declared_repository_url' END, CASE WHEN o.repository_url IS DISTINCT FROM ins.repository_url THEN 'packages.repository_url' END, CASE WHEN o.status IS DISTINCT FROM ins.status THEN 'packages.status' END, - CASE WHEN o.downloads_last_30d IS DISTINCT FROM ins.downloads_last_30d THEN 'packages.downloads_last_30d' END, CASE WHEN o.total_downloads IS DISTINCT FROM ins.total_downloads THEN 'packages.total_downloads' END, CASE WHEN o.dependent_count IS DISTINCT FROM ins.dependent_count THEN 'packages.dependent_count' END, CASE WHEN o.ingestion_source IS DISTINCT FROM ins.ingestion_source THEN 'packages.ingestion_source' END From b1a2ba1c5dcd3939600482f3a813fafaa7c5d940 Mon Sep 17 00:00:00 2001 From: anilb Date: Sat, 18 Jul 2026 16:10:37 +0200 Subject: [PATCH 45/56] fix: sort packagist license arrays deterministically Signed-off-by: anilb --- .../src/packagist/__tests__/normalize.test.ts | 15 +++++++++++++++ .../packages_worker/src/packagist/normalize.ts | 13 +++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts index ce835c0b84..0cbead5fb6 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts @@ -161,6 +161,21 @@ describe('buildPackagistVersionRows', () => { expect(homepage).toBe('https://example.org') }) + it('sorts a multi-license array deterministically, regardless of registry order', () => { + // versions.licenses is documented as deterministically sorted (schema comment) — + // Composer's dual-licensing means the registry can report the same license set in + // different orders across fetches, which must not register as a real change. + const { versionRows, licenses } = buildPackagistVersionRows([ + { + version: '1.0.0', + version_normalized: '1.0.0.0', + license: ['MIT', 'Apache-2.0', 'BSD-3-Clause'], + }, + ]) + expect(versionRows[0].licenses).toEqual(['Apache-2.0', 'BSD-3-Clause', 'MIT']) + expect(licenses).toEqual(['Apache-2.0', 'BSD-3-Clause', 'MIT']) + }) + it('returns a null homepage when the latest version omits or blanks it', () => { const absent = buildPackagistVersionRows([{ version: '1.0.0', version_normalized: '1.0.0.0' }]) expect(absent.homepage).toBeNull() diff --git a/services/apps/packages_worker/src/packagist/normalize.ts b/services/apps/packages_worker/src/packagist/normalize.ts index cd9bb937a7..9f4fc832ba 100644 --- a/services/apps/packages_worker/src/packagist/normalize.ts +++ b/services/apps/packages_worker/src/packagist/normalize.ts @@ -18,6 +18,15 @@ function blankToNull(s: string | null | undefined): string | null { return t || null } +// versions.licenses is documented as a deterministically-sorted SPDX array (schema +// comment, V1779710880__initial_schema.sql) — Composer allows dual/multi-licensed +// releases, so unlike npm/pypi's near-always-single-element input, the same license +// set can arrive from the registry in a different order between fetches. Sorting once +// here keeps semantically identical arrays byte-identical, avoiding false audit noise. +function sortLicenses(licenses: string[] | null | undefined): string[] | null { + return licenses && licenses.length > 0 ? [...licenses].sort() : null +} + export function normalizePackagistStats(pkg: PackagistPackageInfo): NormalizedPackagistStats { const description = blankToNull(pkg.description) const repositoryUrl = blankToNull(pkg.repository) @@ -146,7 +155,7 @@ export function buildPackagistVersionRows(versions: PackagistExpandedVersion[]): publishedAt: v.time ?? null, isLatest: false, // Will be set below isPrerelease: isPackagistPrerelease(v.version_normalized ?? v.version), - licenses: v.license && v.license.length > 0 ? v.license : null, + licenses: sortLicenses(v.license), })) // Find latest: prefer stable over prerelease; within each group use Composer ordering @@ -176,7 +185,7 @@ export function buildPackagistVersionRows(versions: PackagistExpandedVersion[]): const latestReleaseAt = times[times.length - 1] ?? null // Licenses and homepage come from the latest row - const licenses = kept[latestIdx].license?.length ? (kept[latestIdx].license ?? null) : null + const licenses = sortLicenses(kept[latestIdx].license) const homepage = blankToNull(kept[latestIdx].homepage) return { From 9b0aaf5b26d58bdea3a84c37c0baee00de0e6046 Mon Sep 17 00:00:00 2001 From: anilb Date: Sat, 18 Jul 2026 16:11:01 +0200 Subject: [PATCH 46/56] fix: never bump metadata watermark on generic give-up Signed-off-by: anilb --- .../packages_worker/src/packagist/activities.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index 7ae36a9dd1..bc84c439c2 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -363,9 +363,18 @@ export async function ingestPackagistMetadataBatch( qx, candidate.purl, { status: 'error', attempts: attempt, message: String(err) }, - // An item that already succeeded earlier in this same batch's retry sequence - // must not have that success overwritten by an unrelated re-processing failure. - { notBefore: scheduledAt }, + { + // An item that already succeeded earlier in this same batch's retry + // sequence must not have that success overwritten by an unrelated + // re-processing failure. + notBefore: scheduledAt, + // This generic catch-all fires whenever ingestOnePackagistMetadata threw + // (its own classified give-up paths return normally instead) — meaning we + // can't tell whether phase 1 alone succeeded before a transient p2 failure + // exhausted Temporal's retries. Never bump the refresh watermark here, or a + // genuine p2/versions failure gets hidden for the full refresh window. + bumpLastRunAt: false, + }, ), ) From 8c6194bf580138686e29ca08ffe488e30b0fc74e Mon Sep 17 00:00:00 2001 From: anilb Date: Sat, 18 Jul 2026 16:11:19 +0200 Subject: [PATCH 47/56] fix: drain packagist seed response body on error Signed-off-by: anilb --- .../packagist/__tests__/listPackages.test.ts | 21 ++++++++++++++++++ .../src/packagist/fetchPackage.ts | 5 +++-- .../src/packagist/listPackages.ts | 22 +++++++++++++++---- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts b/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts index 5f792a5005..76bfb68d62 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts @@ -115,4 +115,25 @@ describe('fetchPackagistPackageList', () => { ) expect(await fetchPackagistPackageList()).toMatchObject({ kind: 'TRANSIENT', statusCode: 503 }) }) + + // Regression: the seed's own error paths share the same 10-connection dispatcher as + // the stats/p2 fetches, so an undrained body here risks the same socket exhaustion. + it.each([ + ['404', 404, 'NOT_FOUND'], + ['429', 429, 'RATE_LIMIT'], + ['503', 503, 'TRANSIENT'], + ])('drains the response body on a %s error', async (_status, statusCode, kind) => { + const cancel = vi.fn() + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + status: statusCode, + ok: false, + json: async () => ({}), + body: { cancel }, + }), + ) + expect(await fetchPackagistPackageList()).toMatchObject({ kind }) + expect(cancel).toHaveBeenCalled() + }) }) diff --git a/services/apps/packages_worker/src/packagist/fetchPackage.ts b/services/apps/packages_worker/src/packagist/fetchPackage.ts index 12dd3a5159..1589172b69 100644 --- a/services/apps/packages_worker/src/packagist/fetchPackage.ts +++ b/services/apps/packages_worker/src/packagist/fetchPackage.ts @@ -36,8 +36,9 @@ export function describeFetchFailure(err: unknown): string { // An error-path response whose body is never read can pin its socket instead of // returning it to packagistDispatcher's pool (capped at 10 connections per origin) — // canceling it releases the connection for reuse. Best-effort: a failure here must -// never mask the real error already being returned. -async function discardBody(res: Response): Promise { +// never mask the real error already being returned. Exported for listPackages.ts, +// which shares the same dispatcher and error-path risk. +export async function discardBody(res: Response): Promise { try { await res.body?.cancel() } catch { diff --git a/services/apps/packages_worker/src/packagist/listPackages.ts b/services/apps/packages_worker/src/packagist/listPackages.ts index 8e561ed9be..e906d988a6 100644 --- a/services/apps/packages_worker/src/packagist/listPackages.ts +++ b/services/apps/packages_worker/src/packagist/listPackages.ts @@ -1,6 +1,11 @@ import type { Dispatcher } from 'undici' -import { buildPackagistUserAgent, describeFetchFailure, packagistDispatcher } from './fetchPackage' +import { + buildPackagistUserAgent, + describeFetchFailure, + discardBody, + packagistDispatcher, +} from './fetchPackage' import type { FetchError } from './types' const PACKAGIST_LIST = 'https://packagist.org/packages/list.json' @@ -96,9 +101,18 @@ export async function fetchPackagistPackageList(): Promise } // Status classification: 404 NOT_FOUND, 429 RATE_LIMIT, other non-ok TRANSIENT - if (res.status === 404) return { kind: 'NOT_FOUND', message: 'list not found', statusCode: 404 } - if (res.status === 429) return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429 } - if (!res.ok) return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + if (res.status === 404) { + await discardBody(res) + return { kind: 'NOT_FOUND', message: 'list not found', statusCode: 404 } + } + if (res.status === 429) { + await discardBody(res) + return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429 } + } + if (!res.ok) { + await discardBody(res) + return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + } let json: unknown try { From 66c653ec0454eed5cd5ae735485098bbb6edf252 Mon Sep 17 00:00:00 2001 From: anilb Date: Sat, 18 Jul 2026 16:11:29 +0200 Subject: [PATCH 48/56] fix: add packagist-worker to frontend-dev ignore lists Signed-off-by: anilb --- scripts/cli | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cli b/scripts/cli index 7bdfd7e3be..f4a18a92db 100755 --- a/scripts/cli +++ b/scripts/cli @@ -1182,14 +1182,14 @@ while test $# -gt 0; do exit ;; service-restart-fe-dev) - IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") + IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "packagist-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") DEV=1 kill_all_containers service_start exit ;; clean-start-fe-dev) - IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "cache-worker" "categorization-worker" "cron-service" "data-sink-worker" "git-integration" "integration-run-worker" "integration-stream-worker" "nango-webhook-api" "nango-worker" "script-executor-worker" "search-sync-api" "search-sync-worker" "security-best-practices-worker" "snowflake-connectors-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") + IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "cache-worker" "categorization-worker" "cron-service" "data-sink-worker" "git-integration" "integration-run-worker" "integration-stream-worker" "nango-webhook-api" "nango-worker" "script-executor-worker" "search-sync-api" "search-sync-worker" "security-best-practices-worker" "snowflake-connectors-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "packagist-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") CLEAN_START=1 DEV=1 start From 174140d57e7b497b5f14133433d1ca10f6cbdf15 Mon Sep 17 00:00:00 2001 From: anilb Date: Sat, 18 Jul 2026 16:11:38 +0200 Subject: [PATCH 49/56] fix: expand legacy GitLab route keyword list Signed-off-by: anilb --- .../__tests__/canonicalizeRepoUrl.test.ts | 18 ++++++++++++++++++ .../src/utils/canonicalizeRepoUrl.ts | 16 ++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts b/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts index a21906bf80..0388081239 100644 --- a/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts +++ b/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts @@ -60,6 +60,24 @@ describe('canonicalizeRepoUrl', () => { 'https://gitlab.com/group/subgroup/project', 'gitlab', ], + [ + // The reviewer's exact regression example: `raw` was missing from the legacy + // route list, so this previously resolved as the bogus nested repo + // `group/project/raw`. + 'https://gitlab.com/group/project/raw/main/file.php', + 'https://gitlab.com/group/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/project/blame/main/file.php', + 'https://gitlab.com/group/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/subgroup/project/issues/42', + 'https://gitlab.com/group/subgroup/project', + 'gitlab', + ], ])('canonicalizes %s', (input, expectedUrl, expectedHost) => { expect(canonicalizeRepoUrl(input)).toEqual({ url: expectedUrl, host: expectedHost }) }) diff --git a/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts b/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts index 58dfce174a..b08617c940 100644 --- a/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts +++ b/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts @@ -27,17 +27,29 @@ const CASE_INSENSITIVE_HOSTS = new Set(['github.com', 'gitlab.com']) // Pre-2018 GitLab URLs (and shorthand copies of them still circulating) mark deep-links // the same way GitHub does — appended directly after the project path, with no `/-/` // separator. GitLab reserves these route words at the project-slug position precisely so -// they can never collide with a real project name (docs: user/reserved_names), so treating -// the first one as a deep-link boundary is safe even for arbitrarily nested subgroups. +// they can never collide with a real project name (docs: user/reserved_names, the +// PROJECT_WILDCARD_ROUTES list), so treating the first one as a deep-link boundary is +// safe even for arbitrarily nested subgroups. const GITLAB_LEGACY_ROUTE_SEGMENTS = new Set([ 'tree', 'blob', + 'blame', + 'raw', 'commits', 'commit', 'compare', 'issues', 'merge_requests', 'wikis', + 'builds', + 'create', + 'create_dir', + 'edit', + 'find_file', + 'new', + 'preview', + 'refs', + 'update', ]) /** From 0f183c378d9b85e3a253aa8ad3601bc39d0b6974 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 19 Jul 2026 00:14:13 +0200 Subject: [PATCH 50/56] fix: give packagist dev inspector its own port Signed-off-by: anilb --- services/apps/packages_worker/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json index 6cd1537c6c..3d15172900 100644 --- a/services/apps/packages_worker/package.json +++ b/services/apps/packages_worker/package.json @@ -52,8 +52,8 @@ "dev:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts", "dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts", "start:packagist-worker": "CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker tsx src/bin/packagist-worker.ts", - "dev:packagist-worker": "CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/packagist-worker.ts", - "dev:packagist-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/packagist-worker.ts", + "dev:packagist-worker": "CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9245 src/bin/packagist-worker.ts", + "dev:packagist-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9245 src/bin/packagist-worker.ts", "start:rubygems-worker": "CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker tsx src/bin/rubygems-worker.ts", "dev:rubygems-worker": "CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts", "dev:rubygems-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts", From 6f5d90c2a942e4ee9598db1893736c654c5cb682 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 19 Jul 2026 00:14:22 +0200 Subject: [PATCH 51/56] docs: fix stale downloads_last_30d ownership claim Signed-off-by: anilb --- services/apps/packages_worker/src/packagist/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md index 7edb6157b2..29dcec90d2 100644 --- a/services/apps/packages_worker/src/packagist/README.md +++ b/services/apps/packages_worker/src/packagist/README.md @@ -91,9 +91,10 @@ touching p2. - From the **dynamic endpoint**: - `packages` — `description`, `declared_repository_url`, `repository_url` (canonicalized), `status` (`abandoned` → `deprecated`, else `active`), - `downloads_last_30d` (rolling monthly), `total_downloads`, - `dependent_count` (registry-reported direct dependents — a ranking input), - `last_synced_at`. Counters COALESCE: a missing value never wipes a known one. + `total_downloads`, `dependent_count` (registry-reported direct + dependents — a ranking input), `last_synced_at`. Counters COALESCE: a + missing value never wipes a known one. `downloads_last_30d` is + deliberately NOT written here — see the dedicated monthly lane below. - `repos` + `package_repos` — **all packages**: canonicalized repo URL linked with `source='declared'`, `confidence=0.8` (the manifest-declared convention shared with npm/pypi/maven/cargo); feeds the shared github-repos-enricher. From 5b609caec17b3bca3638230fdb46b874e3b75dd2 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 19 Jul 2026 01:05:56 +0200 Subject: [PATCH 52/56] fix: anchor packagist metadata due-selection to a fixed cutoff Signed-off-by: anilb --- .../packagist/__tests__/dueSelection.test.ts | 15 +++++--- .../src/packagist/__tests__/ingest.test.ts | 36 +++++++++++++++---- .../src/packagist/activities.ts | 11 +++++- .../src/packagist/workflows.ts | 14 ++++++-- .../src/packages/packagistPackageState.ts | 6 ++-- 5 files changed, 64 insertions(+), 18 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts index 84999d06e8..3ebc2614e2 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts @@ -35,15 +35,20 @@ beforeEach(() => { }) // Metadata (merged enrichment) due-selection: keyset pagination, refresh window, and -// the stored Last-Modified for If-Modified-Since replay on the p2 fetch. +// the stored Last-Modified for If-Modified-Since replay on the p2 fetch. `cutoff` is a +// pre-computed threshold (this run's fixed start time minus the refresh window), not a +// live NOW() — a keyset scan only visits each purl once per drain, so due-selection +// must stay anchored to one stable point in time for the whole run. describe('getPackagistMetadataDuePurls', () => { - it('returns purls with their stored Last-Modified, scoped by refresh window', async () => { + const CUTOFF = '2026-07-08T00:00:00.000Z' + + it('returns purls with their stored Last-Modified, scoped by the fixed cutoff', async () => { qx.select.mockResolvedValue([ { purl: 'pkg:composer/a/x', metadata_last_modified: 'Tue, 30 Jun 2026 00:00:00 GMT' }, { purl: 'pkg:composer/b/y', metadata_last_modified: null }, ]) - const candidates = await getPackagistMetadataDuePurls(asQx(qx), '', 50, 7, true) + const candidates = await getPackagistMetadataDuePurls(asQx(qx), CUTOFF, '', 50, true) expect(candidates).toEqual([ { purl: 'pkg:composer/a/x', metadataLastModified: 'Tue, 30 Jun 2026 00:00:00 GMT' }, @@ -54,11 +59,11 @@ describe('getPackagistMetadataDuePurls', () => { expect(sql).toMatch(/is_critical/) expect(sql).toMatch(/metadata_last_run_at/) expect(sql).toMatch(/ORDER BY\s+p\.purl/i) - expect(params).toMatchObject({ batchSize: 50, refreshDays: 7, onlyCritical: true }) + expect(params).toMatchObject({ cutoff: CUTOFF, batchSize: 50, onlyCritical: true }) }) it('selects across all packagist packages when onlyCritical is false (the all-packages default)', async () => { - await getPackagistMetadataDuePurls(asQx(qx), '', 50, 7, false) + await getPackagistMetadataDuePurls(asQx(qx), CUTOFF, '', 50, false) const [sql, params] = qx.select.mock.calls[0] expect(sql).toMatch(/ecosystem = 'packagist'/) diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts index 086021a644..cb79a500b5 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -414,29 +414,51 @@ describe('ingestPackagistItemsConcurrently', () => { // All-packages metadata scope: the env default drives due-selection's onlyCritical arg. // Default is ALL packages (deps.dev has no Packagist data to fall back on); the env var -// narrows back to the critical slice. +// narrows back to the critical slice. The due-cutoff is derived once from the run's +// fixed cutoff minus the refresh window, not a live NOW() re-evaluated per batch. describe('getPackagistMetadataBatch scope', () => { const mockMetadataDue = vi.mocked(getPackagistMetadataDuePurls) + const RUN_CUTOFF = '2026-07-15T00:00:00.000Z' + const DUE_CUTOFF = '2026-07-08T00:00:00.000Z' // RUN_CUTOFF minus the 7-day default afterEach(() => { delete process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL + delete process.env.CROWD_PACKAGES_PACKAGIST_METADATA_REFRESH_DAYS + }) + + it('derives the due-cutoff from the fixed run cutoff, not a live clock', async () => { + process.env.CROWD_PACKAGES_PACKAGIST_METADATA_REFRESH_DAYS = '3' + mockMetadataDue.mockResolvedValue([]) + + await getPackagistMetadataBatch(RUN_CUTOFF, '', 50) + + // 3 days before the run's own fixed cutoff — never "3 days before whenever this + // particular batch happens to execute" — is what makes the keyset scan give every + // purl exactly one refresh-eligibility check per drain. + expect(mockMetadataDue).toHaveBeenCalledWith( + expect.anything(), + '2026-07-12T00:00:00.000Z', + '', + 50, + false, + ) }) it('selects across ALL packagist packages by default', async () => { mockMetadataDue.mockResolvedValue([]) - await getPackagistMetadataBatch('', 50) + await getPackagistMetadataBatch(RUN_CUTOFF, '', 50) - expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), '', 50, 7, false) + expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), DUE_CUTOFF, '', 50, false) }) it('narrows to the critical slice when CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL=true', async () => { process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL = 'true' mockMetadataDue.mockResolvedValue([]) - await getPackagistMetadataBatch('', 50) + await getPackagistMetadataBatch(RUN_CUTOFF, '', 50) - expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), '', 50, 7, true) + expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), DUE_CUTOFF, '', 50, true) }) it('keeps the all-packages default when the env value is unrecognized', async () => { @@ -444,9 +466,9 @@ describe('getPackagistMetadataBatch scope', () => { process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL = 'ture' mockMetadataDue.mockResolvedValue([]) - await getPackagistMetadataBatch('', 50) + await getPackagistMetadataBatch(RUN_CUTOFF, '', 50) - expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), '', 50, 7, false) + expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), DUE_CUTOFF, '', 50, false) }) }) diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts index bc84c439c2..ee610e26df 100644 --- a/services/apps/packages_worker/src/packagist/activities.ts +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -323,16 +323,25 @@ export async function runPackagistPackageSeed(): Promise<{ discovered: number; i return { discovered: entries.length, invalid } } +// `cutoff` is the drain's own fixed start time (stable across every round via +// continueAsNew), not a live NOW() — a keyset scan only visits each purl once per +// drain, so re-deriving "7 days ago" fresh on every batch would silently skip a purl +// that hasn't quite hit the refresh window yet when the cursor passes it, pushing its +// effective cadence out toward two refresh cycles instead of one. export async function getPackagistMetadataBatch( + cutoff: string, afterPurl: string, batchSize: number, ): Promise<{ candidates: PackagistMetadataCandidate[]; nextCursor: string }> { const qx = await getPackagesDb() + const dueCutoff = new Date( + new Date(cutoff).getTime() - metadataRefreshDays() * 24 * 60 * 60 * 1000, + ).toISOString() const candidates = await getPackagistMetadataDuePurls( qx, + dueCutoff, afterPurl, batchSize, - metadataRefreshDays(), runOnlyForCritical(), ) return { diff --git a/services/apps/packages_worker/src/packagist/workflows.ts b/services/apps/packages_worker/src/packagist/workflows.ts index 6365a241ab..52be4453eb 100644 --- a/services/apps/packages_worker/src/packagist/workflows.ts +++ b/services/apps/packages_worker/src/packagist/workflows.ts @@ -23,6 +23,7 @@ const INGEST_BATCH = 50 const ROUNDS_PER_RUN = 20 interface MetadataState { + cutoff?: string cursor?: string } @@ -54,12 +55,21 @@ export async function seedPackagistPackages(): Promise { } } +// The cutoff is fixed once per run (deterministic activity), same pattern as the +// downloads-30d/daily lanes — a keyset scan only ever visits each purl once per drain, +// so due-selection must be anchored to a stable point in time rather than a live NOW() +// that would let a purl processed early in the run dodge this cycle's refresh window. export async function ingestPackagistMetadata(state: MetadataState = {}): Promise { + const cutoff = state.cutoff ?? (await acts.packagistCurrentTimestamp()) let cursor = state.cursor || '' const stopAfterFirstPage = await acts.packagistStopAfterFirstPage() for (let r = 0; r < ROUNDS_PER_RUN; r++) { - const { candidates, nextCursor } = await acts.getPackagistMetadataBatch(cursor, INGEST_BATCH) + const { candidates, nextCursor } = await acts.getPackagistMetadataBatch( + cutoff, + cursor, + INGEST_BATCH, + ) if (candidates.length === 0) return await acts.ingestPackagistMetadataBatch(candidates) cursor = nextCursor @@ -67,7 +77,7 @@ export async function ingestPackagistMetadata(state: MetadataState = {}): Promis if (candidates.length < INGEST_BATCH) return } - await continueAsNew({ cursor }) + await continueAsNew({ cutoff, cursor }) } // Monthly capture of the observed rolling 30d window for every packagist package. diff --git a/services/libs/data-access-layer/src/packages/packagistPackageState.ts b/services/libs/data-access-layer/src/packages/packagistPackageState.ts index 3e6a24aaad..8d734a2474 100644 --- a/services/libs/data-access-layer/src/packages/packagistPackageState.ts +++ b/services/libs/data-access-layer/src/packages/packagistPackageState.ts @@ -68,9 +68,9 @@ export async function markPackagistMetadataScanned( export async function getPackagistMetadataDuePurls( qx: QueryExecutor, + cutoff: string, afterPurl: string, batchSize: number, - refreshDays: number, onlyCritical: boolean, ): Promise { const rows: Array<{ purl: string; metadata_last_modified: string | null }> = await qx.select( @@ -82,11 +82,11 @@ export async function getPackagistMetadataDuePurls( AND p.purl > $(afterPurl) AND ( s.metadata_last_run_at IS NULL - OR s.metadata_last_run_at < NOW() - ($(refreshDays) || ' days')::interval + OR s.metadata_last_run_at < $(cutoff)::timestamptz ) ORDER BY p.purl LIMIT $(batchSize)`, - { afterPurl, batchSize, refreshDays, onlyCritical }, + { cutoff, afterPurl, batchSize, onlyCritical }, ) return rows.map((r) => ({ purl: r.purl, From 9e72e3cde2368a9aa80973c82caf4624ad48fd02 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 19 Jul 2026 01:06:05 +0200 Subject: [PATCH 53/56] fix: never downgrade repo link provenance from packagist Signed-off-by: anilb --- .../__tests__/persistPackageInfo.test.ts | 6 +-- .../src/packagist/upsertPackageInfo.ts | 18 +++++---- .../data-access-layer/src/packages/repos.ts | 39 +++++++++++++++++++ 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts index 77eb4d6e58..69bbacdd98 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts @@ -6,7 +6,7 @@ import { removeDeclaredPackageRepo, updatePackagistPackageStats, upsertPackageMaintainers, - upsertPackageRepo, + upsertPackageRepoPreserveProvenance, } from '@crowd/data-access-layer/src/packages' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' @@ -17,7 +17,7 @@ vi.mock('@crowd/data-access-layer/src/packages', () => ({ updatePackagistPackageStats: vi.fn(), upsertPackageMaintainers: vi.fn().mockResolvedValue([]), getOrCreateRepoByUrl: vi.fn(), - upsertPackageRepo: vi.fn().mockResolvedValue([]), + upsertPackageRepoPreserveProvenance: vi.fn().mockResolvedValue([]), removeDeclaredPackageRepo: vi.fn().mockResolvedValue([]), logAuditFieldChanges: vi.fn(), })) @@ -25,7 +25,7 @@ vi.mock('@crowd/data-access-layer/src/packages', () => ({ const mockUpdate = vi.mocked(updatePackagistPackageStats) const mockMaintainers = vi.mocked(upsertPackageMaintainers) const mockRepoGet = vi.mocked(getOrCreateRepoByUrl) -const mockRepoLink = vi.mocked(upsertPackageRepo) +const mockRepoLink = vi.mocked(upsertPackageRepoPreserveProvenance) const mockRepoRemove = vi.mocked(removeDeclaredPackageRepo) const mockAudit = vi.mocked(logAuditFieldChanges) diff --git a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts index 860623fb3d..a975f4cb56 100644 --- a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts +++ b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts @@ -4,7 +4,7 @@ import { removeDeclaredPackageRepo, updatePackagistPackageStats, upsertPackageMaintainers, - upsertPackageRepo, + upsertPackageRepoPreserveProvenance, } from '@crowd/data-access-layer/src/packages' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' @@ -61,14 +61,18 @@ export async function persistPackagistPackageInfo( changedFields.push(...result.changedFields) // Step 2: Link the repo for ALL packages — 'declared'/0.8 is the manifest-declared - // convention shared by npm/pypi/maven/cargo. When there's no trusted repo (removed - // from the manifest, or no longer canonicalizable to a known host), or it now - // resolves to a different repo, clear any previously-declared link that no longer - // applies — package_repos' unique key is (package_id, repo_id), not (package_id, - // source), so upserting the new link alone would leave a stale one dangling. + // convention shared by npm/pypi/maven/cargo. Preserve-provenance: a higher-confidence + // link another pipeline (manual/deps_dev) already owns for this repo must not be + // downgraded to 'declared'/0.8 by a routine weekly refresh — matches + // upsertMavenPackageRepo/cargo's established GREATEST-confidence, source-preserving + // pattern. When there's no trusted repo (removed from the manifest, or no longer + // canonicalizable to a known host), or it now resolves to a different repo, clear any + // previously-declared link that no longer applies — package_repos' unique key is + // (package_id, repo_id), not (package_id, source), so upserting the new link alone + // would leave a stale one dangling. if (trustedRepo) { const repo = await getOrCreateRepoByUrl(t, trustedRepo.url, trustedRepo.host) - const linkChanged = await upsertPackageRepo(t, id, repo.id, 'declared', 0.8) + const linkChanged = await upsertPackageRepoPreserveProvenance(t, id, repo.id, 'declared', 0.8) const removedFields = await removeDeclaredPackageRepo(t, id, repo.id) changedFields.push(...repo.changedFields, ...linkChanged, ...removedFields) } else { diff --git a/services/libs/data-access-layer/src/packages/repos.ts b/services/libs/data-access-layer/src/packages/repos.ts index 362415b8ac..92d14a55b8 100644 --- a/services/libs/data-access-layer/src/packages/repos.ts +++ b/services/libs/data-access-layer/src/packages/repos.ts @@ -87,3 +87,42 @@ export async function upsertPackageRepo( ) return row.changed_fields } + +// Same shape as upsertPackageRepo, but never lets this write downgrade an existing +// link: `source` is left untouched on conflict (a manual/deps_dev link keeps its +// provenance instead of being reassigned to this caller's source) and `confidence` +// only ever moves up via GREATEST. Matches the established pattern in +// upsertMavenPackageRepo (osspckgs/repos.ts) and cargo/enrich.ts's inline equivalent — +// added here as a separate function rather than changing upsertPackageRepo itself, +// which pypi/npm/go/nuget/rubygems also call and rely on staying an unconditional set. +export async function upsertPackageRepoPreserveProvenance( + qx: QueryExecutor, + packageId: string, + repoId: string, + source: string, + confidence: number, +): Promise { + const row: { changed_fields: string[] } = await qx.selectOne( + `WITH old AS ( + SELECT source, confidence FROM package_repos + WHERE package_id = $(packageId)::bigint AND repo_id = $(repoId)::bigint + ), + ins AS ( + INSERT INTO package_repos (package_id, repo_id, source, confidence, created_at) + VALUES ($(packageId)::bigint, $(repoId)::bigint, $(source), $(confidence), NOW()) + ON CONFLICT (package_id, repo_id) DO UPDATE SET + confidence = GREATEST(EXCLUDED.confidence, package_repos.confidence), + verified_at = NOW() + RETURNING source, confidence + ) + SELECT array_remove(ARRAY[ + CASE WHEN o.source IS NULL THEN 'package_repos.repo_id' END, + CASE WHEN o.source IS NULL THEN 'package_repos.source' END, + CASE WHEN o.source IS NULL + OR o.confidence IS DISTINCT FROM ins.confidence THEN 'package_repos.confidence' END + ], NULL) AS changed_fields + FROM ins LEFT JOIN old o ON true`, + { packageId, repoId, source, confidence }, + ) + return row.changed_fields +} From 5c0e3e24265a26a4c5472885a7e037449597eb38 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 19 Jul 2026 01:06:14 +0200 Subject: [PATCH 54/56] fix: validate packagist time field is a parseable date Signed-off-by: anilb --- .../src/packagist/__tests__/fetchPackage.test.ts | 1 + .../apps/packages_worker/src/packagist/fetchPackage.ts | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts b/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts index 79783c20df..ac6cc3fde4 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts @@ -226,6 +226,7 @@ describe('fetchPackagistP2', () => { ['license is a scalar string, not an array', [{ version: '1.0.0', license: 'MIT' }]], ['homepage is an object', [{ version: '1.0.0', homepage: {} }]], ['time is a number', [{ version: '1.0.0', time: 20240101 }]], + ['time is a string but not a parseable date', [{ version: '1.0.0', time: 'not-a-date' }]], ])('maps a malformed version entry (%s) → MALFORMED, not a throw', async (_desc, versions) => { vi.stubGlobal( 'fetch', diff --git a/services/apps/packages_worker/src/packagist/fetchPackage.ts b/services/apps/packages_worker/src/packagist/fetchPackage.ts index 1589172b69..28b4d3326e 100644 --- a/services/apps/packages_worker/src/packagist/fetchPackage.ts +++ b/services/apps/packages_worker/src/packagist/fetchPackage.ts @@ -198,9 +198,11 @@ function isValidVersionField(value: unknown, isValidType: (v: unknown) => boolea // `version`/`version_normalized` via .startsWith()/.split()/.endsWith() // (isPackagistDevVersion/isPackagistPrerelease), `homepage` via blankToNull's .trim(), // `license` as the text[] written to versions.licenses, `time` as the timestamptz -// written to versions.published_at. `require`/`require-dev` are already guarded at -// their own call site (extractVersionDependencies checks typeof before Object.entries) -// so aren't re-validated here. +// written to versions.published_at — checking it's merely a string isn't enough, since +// a non-date string like "not-a-date" would still be typeof 'string' but throw at the +// v.pub::timestamptz SQL cast, deep past this guard. `require`/`require-dev` are already +// guarded at their own call site (extractVersionDependencies checks typeof before +// Object.entries) so aren't re-validated here. function isValidMinifiedVersion(v: unknown): boolean { if (typeof v !== 'object' || v === null) return false const entry = v as Record @@ -208,7 +210,7 @@ function isValidMinifiedVersion(v: unknown): boolean { typeof entry.version === 'string' && isValidVersionField(entry.version_normalized, (x) => typeof x === 'string') && isValidVersionField(entry.homepage, (x) => typeof x === 'string') && - isValidVersionField(entry.time, (x) => typeof x === 'string') && + isValidVersionField(entry.time, (x) => typeof x === 'string' && !isNaN(Date.parse(x))) && isValidVersionField( entry.license, (x) => Array.isArray(x) && x.every((l) => typeof l === 'string'), From 45c07096079fd67054cfd981fc75d57abaca6dd9 Mon Sep 17 00:00:00 2001 From: anilb Date: Wed, 22 Jul 2026 12:42:30 +0200 Subject: [PATCH 55/56] fix: drop dead composer alias, export FETCHERS for wiring test Signed-off-by: anilb --- .../src/security-contacts/extractors/registry/index.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts b/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts index 7453a9987a..e183aff207 100644 --- a/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts +++ b/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts @@ -17,8 +17,10 @@ type EcosystemFetcher = ( repoUrl?: string, ) => Promise -// Keyed by the lowercased packages.ecosystem value. -const FETCHERS: Record = { +// Keyed by the lowercased packages.ecosystem value. Composer packages live under +// 'packagist' — the value the packagist worker seeds — not 'composer', which no +// writer ever produces (the original entry predated the worker and guessed wrong). +export const FETCHERS: Record = { npm: fetchNpm, pypi: fetchPypi, maven: fetchMaven, @@ -26,7 +28,6 @@ const FETCHERS: Record = { nuget: fetchNuget, rubygems: fetchRubygems, packagist: fetchComposer, - composer: fetchComposer, go: fetchGo, } From 7637434f5c9be376b5a7290f6e0d01bcc9ea05f6 Mon Sep 17 00:00:00 2001 From: anilb Date: Wed, 22 Jul 2026 12:42:39 +0200 Subject: [PATCH 56/56] fix: audit stale is_latest rows cleared outside the batch Signed-off-by: anilb --- .../packagist/__tests__/dueSelection.test.ts | 33 +++++++++++++++++++ .../src/packages/versions.ts | 11 +++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts index 3ebc2614e2..af1d616e82 100644 --- a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts +++ b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts @@ -13,6 +13,7 @@ import { markPackagistDailyProcessed, markPackagistMetadataScanned, } from '@crowd/data-access-layer/src/packages/packagistPackageState' +import { upsertPackagistVersions } from '@crowd/data-access-layer/src/packages/versions' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' // Executes the real DAL functions against a capturing stand-in executor — no database. @@ -256,3 +257,35 @@ describe('getCriticalPackagistPackageCount', () => { expect(sql).toMatch(/is_critical/) }) }) + +// The stale-is_latest cleanup only ever flips rows OUTSIDE the upsert batch (batch rows +// already had is_latest set by the upsert CTE), so its row count must feed changedFields +// separately or those real changes never reach the audit log. +describe('upsertPackagistVersions is_latest cleanup audit', () => { + const versions = [ + { number: '2.0.0', publishedAt: null, isLatest: true, isPrerelease: false, licenses: null }, + ] + + it("appends versions.is_latest when the cleanup cleared a stale row the CTE diff can't see", async () => { + qx.selectOne.mockResolvedValue({ changed_fields: [], version_ids: [] }) + qx.result.mockResolvedValue(1) + + const { changedFields } = await upsertPackagistVersions(asQx(qx), '7', versions, '2.0.0') + + const [cleanupSql] = qx.result.mock.calls[0] + expect(cleanupSql).toMatch(/is_latest = false/) + expect(changedFields).toContain('versions.is_latest') + }) + + it('reports nothing extra when the cleanup cleared no rows, and never duplicates the CTE diff', async () => { + qx.selectOne.mockResolvedValue({ changed_fields: ['versions.is_latest'], version_ids: [] }) + qx.result.mockResolvedValue(0) + + const noClear = await upsertPackagistVersions(asQx(qx), '7', versions, '2.0.0') + expect(noClear.changedFields).toEqual(['versions.is_latest']) + + qx.result.mockResolvedValue(2) + const cleared = await upsertPackagistVersions(asQx(qx), '7', versions, '2.0.0') + expect(cleared.changedFields.filter((f) => f === 'versions.is_latest')).toHaveLength(1) + }) +}) diff --git a/services/libs/data-access-layer/src/packages/versions.ts b/services/libs/data-access-layer/src/packages/versions.ts index 523454da80..f734709503 100644 --- a/services/libs/data-access-layer/src/packages/versions.ts +++ b/services/libs/data-access-layer/src/packages/versions.ts @@ -155,17 +155,24 @@ export async function upsertPackagistVersions( // records were updated differently can't keep is_latest = true alongside the new latest. When // no latest is known, leave flags untouched rather than wipe all. last_synced_at must move too // — it's the Tinybird ENGINE_VER, and this row's is_latest is a real change. + const changedFields = [...row.changed_fields] if (latestNumber != null) { - await qx.result( + const cleared = await qx.result( `UPDATE versions SET is_latest = false, last_synced_at = NOW() WHERE package_id = $(packageId)::bigint AND is_latest = true AND number <> $(latestNumber)`, { packageId, latestNumber }, ) + // The cleanup only ever flips rows OUTSIDE this batch (batch rows already got their + // is_latest set by the upsert above, so `is_latest = true` skips them) — the CTE diff + // can't see these changes, so surface them for the audit log here. + if (cleared > 0 && !changedFields.includes('versions.is_latest')) { + changedFields.push('versions.is_latest') + } } - return { changedFields: row.changed_fields, versionIds } + return { changedFields, versionIds } } export interface PypiVersionInput {