From 7df1ff9ee01dc6de68e21abec2427fc9686ed808 Mon Sep 17 00:00:00 2001 From: Nicholas Lim Date: Wed, 1 Jul 2026 18:31:12 +0800 Subject: [PATCH 1/8] docs(export): add NLDS export ADR, spec, and implementation plan Co-Authored-By: Claude Opus 4.8 (1M context) --- ...s-export-as-bundled-script-in-app-image.md | 133 +++ docs/decisions/README.md | 1 + .../plans/2026-07-01-nlds-export.md | 821 ++++++++++++++++++ .../specs/2026-07-01-nlds-export-design.md | 353 ++++++++ 4 files changed, 1308 insertions(+) create mode 100644 docs/decisions/0006-package-nlds-export-as-bundled-script-in-app-image.md create mode 100644 docs/superpowers/plans/2026-07-01-nlds-export.md create mode 100644 docs/superpowers/specs/2026-07-01-nlds-export-design.md diff --git a/docs/decisions/0006-package-nlds-export-as-bundled-script-in-app-image.md b/docs/decisions/0006-package-nlds-export-as-bundled-script-in-app-image.md new file mode 100644 index 00000000..185edf6d --- /dev/null +++ b/docs/decisions/0006-package-nlds-export-as-bundled-script-in-app-image.md @@ -0,0 +1,133 @@ +--- +status: 'accepted' +date: 2026-07-01 +decision-makers: [Onward app team] +consulted: [] +informed: [Transform infra team] +--- + +# Package the NLDS export as a bundled script inside the existing app image + +## Context and Problem Statement + +A daily scheduled ECS task must run a batch export from the Onward database to +the NLDS-owned S3 data lake. The infra side is already drafted around +**reusing the learner (app) image** and invoking `pnpm run export:nlds`; the task +runs to completion and exits. + +The complication is what actually ships in that image. The production Docker stage +copies only `build/`, `prisma/`, `package.json`, and the pruned **production** +`node_modules`. It does **not** copy `src/`, and it does **not** include +`tsx`/`dotenv` (both devDependencies). Two consequences follow: the +`node --import=tsx` pattern used by the seed script cannot run in the container, +and the generated Prisma client — emitted as **TypeScript** under +`src/generated/prisma/` by the Prisma 7 `prisma-client` generator — is absent +from the image except where the app's build has already bundled it into `build/`. +`pg` is likewise not directly importable under pnpm's strict layout, since it is +only a transitive dependency of the Postgres driver adapter. + +So the export cannot be a raw TypeScript file executed by an interpreter at +runtime. Some form of ahead-of-time compilation is required, and we must decide +where that artifact lives and how the task reaches it. + +## Decision Drivers + +- **Honor the fixed infra contract** — reuse the learner image; command is + `pnpm run export:nlds`; exit code drives alerting. +- **Add no new runtime dependencies** — the constraint is Prisma plus the AWS S3 + client only. +- **Minimize image bloat and attack surface** — the same image also serves the + live app; anything added to the runtime stage is exposed there too. +- **Keep local execution viable** — a developer must be able to run the export + against a dev database and a test prefix (the acceptance check). +- **Low operational overhead** — avoid new build/publish pipelines and image + lifecycles where possible. + +## Considered Options + +- Compile the export ahead of time into the app's build output and run the + resulting plain-JavaScript artifact with the bare Node runtime. +- Ship the TypeScript source plus a TypeScript runtime in the runtime image and + execute the source directly. +- Build and publish a separate, dedicated image for the export task. + +## Decision Outcome + +Chosen option: **compile the export ahead of time into the app's build output and +run the resulting plain-JavaScript artifact with the bare Node runtime**, because +it is the only option that satisfies every driver at once — it reuses the +existing image and the fixed command, adds no runtime dependencies, and adds only +a single inert bundled file to the runtime stage. + +The bundling runs in the **build stage** using the bundler that already ships with +the toolchain, so it introduces no dependency into the runtime stage. The +generated Prisma client is inlined into the bundle, which resolves both the +missing-`src/` problem and the transitive-`pg` problem. The runtime stage gains +one self-contained JavaScript file, which is inert until the scheduled task +invokes it. + +### Consequences + +- Good, because the runtime image gains no new packages — the bundler is + build-stage only — so the marginal attack surface over the app image is + effectively nil. +- Good, because it keeps the drafted infra untouched: same image, same + `pnpm run export:nlds` entrypoint, exit-code semantics preserved. +- Good, because inlining the generated client sidesteps the TypeScript-source and + strict-`pg`-resolution problems without loosening pnpm's layout. +- Bad, because it adds a bundling step to the Docker build and a matching + dev-only bundler dependency, which must be kept working as the build evolves. +- Bad, because the bundled artifact is a second build product to reason about; a + Prisma-generator or schema change is only reflected after a rebuild. +- Neutral, because local runs bundle-then-execute (or use the interpreter path + available in dev), a minor divergence from the container's execute-only step. + +### Confirmation + +Running the export against a dev database and a test prefix produces the expected +S3 objects and exits zero; a forced failure exits non-zero. In the built image, +the entrypoint resolves and executes with only the production dependency set +present (no TypeScript runtime, no `src/`). + +## Pros and Cons of the Options + +### Compile ahead of time into the build output; run plain JS + +- Good, because no runtime dependency is added; the bundler is confined to the + build stage. +- Good, because only one inert file is added to the runtime image. +- Good, because it reuses the fixed image and command with no infra change. +- Neutral, because local execution needs a bundle step (or the dev interpreter), + a small workflow wrinkle. +- Bad, because it couples the export artifact to the Docker build and requires a + rebuild to pick up schema/generator changes. + +### Ship TypeScript source plus a TypeScript runtime in the image + +- Good, because it is the least code to author and matches the existing seed + workflow one-to-one. +- Bad, because it promotes a TypeScript interpreter (and `dotenv`) to runtime + dependencies, directly violating the "Prisma + S3 client only" constraint. +- Bad, because it enlarges the runtime image and its attack surface, on an image + that also serves the live app. +- Bad, because it requires copying additional source directories into the runtime + stage. + +### Separate dedicated image for the export + +- Good, because it isolates the export's lifecycle from the app image. +- Bad, because it contradicts the already-drafted infra (which reuses the learner + image) and forces a new registry repository, a second build-and-publish + pipeline, and a task-definition image change. +- Bad, because it does not shrink the dependency set — the export needs the same + Prisma engine, Postgres driver, and S3 client — so the same surface is simply + rebuilt elsewhere. +- Neutral, because the isolation benefit only matters if the export later needs an + independent release cadence, which is out of scope now. + +## More Information + +The dataset selection, output format, S3 key layout, and encryption requirements +are specified separately in the design spec that consumes this decision +(`docs/superpowers/specs/2026-07-01-nlds-export-design.md`). This ADR governs only +how the export is packaged and executed, not what it exports. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 37adc14e..1e245270 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -39,3 +39,4 @@ the ADR; the spec consumes its outcome. - [0002 — Read streaming exports with keyset cursor pagination on the primary key](./0002-keyset-cursor-pagination-on-primary-key.md) — accepted - [0003 — Nested per-report download endpoints and separate per-report page routes](./0003-report-export-routing-and-page-routes.md) — accepted - [0004 — Card-grid report index with per-page back navigation](./0004-card-grid-report-index.md) — accepted +- [0006 — Package the NLDS export as a bundled script inside the existing app image](./0006-package-nlds-export-as-bundled-script-in-app-image.md) — accepted diff --git a/docs/superpowers/plans/2026-07-01-nlds-export.md b/docs/superpowers/plans/2026-07-01-nlds-export.md new file mode 100644 index 00000000..97270d0f --- /dev/null +++ b/docs/superpowers/plans/2026-07-01-nlds-export.md @@ -0,0 +1,821 @@ +# NLDS Mandatory-Quiz-Completion Export Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a daily batch script that exports a full snapshot of users and mandatory-module quiz outcomes from the Onward DB to the NLDS S3 data lake as NDJSON, encrypted with SSE-KMS. + +**Architecture:** A single standalone entry module `scripts/export-nlds.ts` implements small, individually-testable units (config loader, dataset definitions, NDJSON serializer, S3 uploader, orchestrator). It instantiates its own Prisma client over `@prisma/adapter-pg` (mirroring `prisma/seed.ts`, not the SvelteKit `$lib/server/db` module). Per [ADR-0006](../../decisions/0006-package-nlds-export-as-bundled-script-in-app-image.md) it is bundled at Docker build time to `build/export-nlds.js` (generated Prisma client inlined; AWS SDK + Prisma runtime left external) and run by bare Node. + +**Tech Stack:** TypeScript, Node 24 (ESM), Prisma 7 + `@prisma/adapter-pg`, `@aws-sdk/client-s3`, esbuild (build-stage bundler, devDep), Vitest. + +## Global Constraints + +- **Runtime deps:** only Prisma and `@aws-sdk/client-s3` may be used at runtime. `esbuild` is a **devDependency**, build-stage only — never shipped to the production image stage. Do NOT import `dotenv` or `tsx` in the script. +- **Invocation contract (fixed by infra):** command is `pnpm run export:nlds`; exit `0` on full success, non-zero on any failure; Node/ARM64, no shell tools. +- **Env vars (injected):** `POSTGRES_URL`, `NLDS_BUCKET`, `NLDS_PREFIX` (e.g. `glow/`), `NLDS_KMS_KEY_ID`, `AWS_REGION`. All required; no defaults. +- **SSE-KMS mandatory** on every `PutObject`: `ServerSideEncryption: 'aws:kms'` + `SSEKMSKeyId`. The bucket rejects non-KMS puts. +- **Prefix scoping:** never construct an S3 key outside `NLDS_PREFIX`. +- **Read-only DB:** `select` queries only; never write the source DB. No `SELECT *` — explicit columns only. +- **Output:** NDJSON (one `JSON.stringify` per line, `\n`-terminated incl. last line, UTF-8). Keys: `${NLDS_PREFIX}users/${YYYY-MM-DD}.ndjson` and `${NLDS_PREFIX}mandatory_quiz_outcomes/${YYYY-MM-DD}.ndjson`, date in **UTC**. +- **Commit style:** Conventional Commits (`feat:`, `chore:`, `docs:`). End every commit message with the trailer `Co-Authored-By: Claude Opus 4.8 (1M context) `. +- **Prisma query args** order keys in SQL clause order (`select`, `where`, `orderBy`, …); type args with generated `*FindManyArgs` via `satisfies` and derive rows with `*GetPayload`. + +## GitHub Workflow (document-only; run only when the user explicitly asks) + +No tracking issue exists for this work. The gh steps, in order: + +1. Create the feature branch: `git checkout -b feat/nlds-export` +2. After Task 1's first commit, push and open a **draft** PR: + `gh pr create --draft --title "feat: NLDS mandatory-quiz-completion export" --body-file ` (body per `.github/PULL_REQUEST_TEMPLATE.md`). +3. Final step (after Task 6 passes): `gh pr ready`. + +Writing these into the plan is not running them. Create no branch/commit/push/PR until the user explicitly asks. + +--- + +### Task 1: Config loader + key/date helpers (pure functions) + +Establishes the module file and the three pure, I/O-free units. This task also carries the initial project wiring (the test file) since later tasks build on these exports. + +**Files:** + +- Create: `scripts/export-nlds.ts` +- Test: `scripts/export-nlds.test.ts` + +**Interfaces:** + +- Consumes: nothing. +- Produces: + - `interface ExportConfig { postgresUrl: string; bucket: string; prefix: string; kmsKeyId: string; region: string; }` + - `loadConfig(env: NodeJS.ProcessEnv): ExportConfig` + - `objectKey(prefix: string, dataset: string, date: string): string` + - `formatDateUtc(now: Date): string` + +- [ ] **Step 1: Write the failing tests** + +Create `scripts/export-nlds.test.ts`: + +```ts +import { describe, expect, test } from 'vitest'; + +import { formatDateUtc, loadConfig, objectKey } from './export-nlds.js'; + +const fullEnv = { + POSTGRES_URL: 'postgresql://u:p@localhost:5432/db', + NLDS_BUCKET: 'nlds-bucket', + NLDS_PREFIX: 'glow/', + NLDS_KMS_KEY_ID: 'arn:aws:kms:ap-southeast-1:1:key/abc', + AWS_REGION: 'ap-southeast-1', +} satisfies NodeJS.ProcessEnv; + +describe('loadConfig', () => { + test('returns a fully-populated config when all vars are present', () => { + expect(loadConfig(fullEnv)).toEqual({ + postgresUrl: 'postgresql://u:p@localhost:5432/db', + bucket: 'nlds-bucket', + prefix: 'glow/', + kmsKeyId: 'arn:aws:kms:ap-southeast-1:1:key/abc', + region: 'ap-southeast-1', + }); + }); + + test('throws naming the missing var', () => { + const { NLDS_KMS_KEY_ID, ...missing } = fullEnv; + expect(() => loadConfig(missing)).toThrow('NLDS_KMS_KEY_ID'); + }); + + test('treats an empty-string var as missing', () => { + expect(() => loadConfig({ ...fullEnv, NLDS_BUCKET: '' })).toThrow('NLDS_BUCKET'); + }); +}); + +describe('objectKey', () => { + test('composes prefix + dataset + date + .ndjson', () => { + expect(objectKey('glow/', 'users', '2026-07-01')).toBe('glow/users/2026-07-01.ndjson'); + }); +}); + +describe('formatDateUtc', () => { + test('formats as UTC YYYY-MM-DD', () => { + expect(formatDateUtc(new Date('2026-07-01T13:30:00Z'))).toBe('2026-07-01'); + }); + + test('uses UTC, not local time, near a day boundary', () => { + expect(formatDateUtc(new Date('2026-07-01T23:30:00Z'))).toBe('2026-07-01'); + }); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `pnpm test run scripts/export-nlds.test.ts` +Expected: FAIL — cannot resolve `./export-nlds.ts` / exports not defined. + +- [ ] **Step 3: Create the module with the three units** + +Create `scripts/export-nlds.ts`: + +```ts +export interface ExportConfig { + postgresUrl: string; + bucket: string; + prefix: string; + kmsKeyId: string; + region: string; +} + +const REQUIRED_VARS = [ + 'POSTGRES_URL', + 'NLDS_BUCKET', + 'NLDS_PREFIX', + 'NLDS_KMS_KEY_ID', + 'AWS_REGION', +] as const; + +function requireVar(env: NodeJS.ProcessEnv, name: (typeof REQUIRED_VARS)[number]): string { + const value = env[name]; + if (!value) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +} + +export function loadConfig(env: NodeJS.ProcessEnv): ExportConfig { + return { + postgresUrl: requireVar(env, 'POSTGRES_URL'), + bucket: requireVar(env, 'NLDS_BUCKET'), + prefix: requireVar(env, 'NLDS_PREFIX'), + kmsKeyId: requireVar(env, 'NLDS_KMS_KEY_ID'), + region: requireVar(env, 'AWS_REGION'), + }; +} + +export function objectKey(prefix: string, dataset: string, date: string): string { + return `${prefix}${dataset}/${date}.ndjson`; +} + +export function formatDateUtc(now: Date): string { + return now.toISOString().slice(0, 10); +} +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `pnpm test run scripts/export-nlds.test.ts` +Expected: PASS (6 tests). + +- [ ] **Step 5: Commit** + +```bash +git add scripts/export-nlds.ts scripts/export-nlds.test.ts +git commit -m "feat(export): add NLDS export config loader and key helpers + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +### Task 2: NDJSON serializer + +**Files:** + +- Modify: `scripts/export-nlds.ts` +- Test: `scripts/export-nlds.test.ts` + +**Interfaces:** + +- Consumes: nothing. +- Produces: `toNdjson(rows: ReadonlyArray>): string` + +- [ ] **Step 1: Write the failing tests** + +Append to `scripts/export-nlds.test.ts` (add `toNdjson` to the existing import from `./export-nlds.ts`): + +```ts +describe('toNdjson', () => { + test('serializes each row as its own newline-terminated JSON line', () => { + const out = toNdjson([{ a: 1 }, { a: 2 }]); + expect(out).toBe('{"a":1}\n{"a":2}\n'); + }); + + test('returns empty string for no rows', () => { + expect(toNdjson([])).toBe(''); + }); + + test('preserves null fields (does not coerce to empty string)', () => { + expect(toNdjson([{ a: null }])).toBe('{"a":null}\n'); + }); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `pnpm test run scripts/export-nlds.test.ts -t toNdjson` +Expected: FAIL — `toNdjson` is not exported. + +- [ ] **Step 3: Implement `toNdjson`** + +Add to `scripts/export-nlds.ts`: + +```ts +export function toNdjson(rows: ReadonlyArray>): string { + return rows.map((row) => `${JSON.stringify(row)}\n`).join(''); +} +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `pnpm test run scripts/export-nlds.test.ts -t toNdjson` +Expected: PASS (3 tests). + +- [ ] **Step 5: Commit** + +```bash +git add scripts/export-nlds.ts scripts/export-nlds.test.ts +git commit -m "feat(export): add NDJSON serializer + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +### Task 3: Dataset definitions (users + mandatory quiz outcomes) + +**Files:** + +- Modify: `scripts/export-nlds.ts` +- Test: `scripts/export-nlds.test.ts` + +**Interfaces:** + +- Consumes: `PrismaClient` type from `../src/generated/prisma/client.js`. +- Produces: + - `interface Dataset> { readonly name: string; fetch(client: PrismaClient): Promise; }` + - `usersDataset: Dataset` (name `'users'`; `select` id, name, email, created_at) + - `mandatoryQuizOutcomesDataset: Dataset` (name `'mandatory_quiz_outcomes'`; journeys where `learningUnit.is_required = true`) + - `datasets: ReadonlyArray>>` + +- [ ] **Step 1: Write the failing tests** + +Append to `scripts/export-nlds.test.ts`. Add `vi` to the vitest import (`import { describe, expect, test, vi } from 'vitest';`) and add `mandatoryQuizOutcomesDataset, usersDataset` to the `./export-nlds.ts` import. + +```ts +describe('usersDataset', () => { + test('selects exactly the four identity fields and maps to snake_case rows', async () => { + const findMany = vi + .fn() + .mockResolvedValue([ + { id: 'u1', name: 'Ada', email: 'ada@x.gov', createdAt: new Date('2026-01-02T00:00:00Z') }, + ]); + const client = { + user: { findMany }, + } as unknown as import('../src/generated/prisma/client.js').PrismaClient; + + const rows = await usersDataset.fetch(client); + + expect(findMany).toHaveBeenCalledWith({ + select: { id: true, name: true, email: true, createdAt: true }, + }); + expect(rows).toEqual([ + { id: 'u1', name: 'Ada', email: 'ada@x.gov', created_at: '2026-01-02T00:00:00.000Z' }, + ]); + }); +}); + +describe('mandatoryQuizOutcomesDataset', () => { + test('filters to required units and projects the nine fields as snake_case', async () => { + const findMany = vi.fn().mockResolvedValue([ + { + userId: 'u1', + learningUnitId: 'lu1', + isCompleted: true, + isQuizAttempted: true, + isQuizPassed: null, + numberOfAttempts: 2, + updatedAt: new Date('2026-06-30T10:00:00Z'), + learningUnit: { title: 'Cyber Hygiene', dueDate: new Date('2026-12-31T00:00:00Z') }, + }, + ]); + const client = { + learningJourney: { findMany }, + } as unknown as import('../src/generated/prisma/client.js').PrismaClient; + + const rows = await mandatoryQuizOutcomesDataset.fetch(client); + + expect(findMany).toHaveBeenCalledWith({ + select: { + userId: true, + learningUnitId: true, + isCompleted: true, + isQuizAttempted: true, + isQuizPassed: true, + numberOfAttempts: true, + updatedAt: true, + learningUnit: { select: { title: true, dueDate: true } }, + }, + where: { learningUnit: { isRequired: true } }, + }); + expect(rows).toEqual([ + { + user_id: 'u1', + learning_unit_id: 'lu1', + learning_unit_title: 'Cyber Hygiene', + due_date: '2026-12-31', + is_completed: true, + is_quiz_attempted: true, + is_quiz_passed: null, + number_of_attempts: 2, + updated_at: '2026-06-30T10:00:00.000Z', + }, + ]); + }); + + test('emits null due_date when the unit has none', async () => { + const findMany = vi.fn().mockResolvedValue([ + { + userId: 'u1', + learningUnitId: 'lu1', + isCompleted: false, + isQuizAttempted: false, + isQuizPassed: null, + numberOfAttempts: 0, + updatedAt: new Date('2026-06-30T10:00:00Z'), + learningUnit: { title: 'AI Literacy', dueDate: null }, + }, + ]); + const client = { + learningJourney: { findMany }, + } as unknown as import('../src/generated/prisma/client.js').PrismaClient; + + const [row] = await mandatoryQuizOutcomesDataset.fetch(client); + + expect(row.due_date).toBeNull(); + }); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `pnpm test run scripts/export-nlds.test.ts -t Dataset` +Expected: FAIL — datasets not exported. + +- [ ] **Step 3: Implement the datasets** + +Add to `scripts/export-nlds.ts` (put the type import with the other imports at the top of the file): + +```ts +import { type PrismaClient } from '../src/generated/prisma/client.js'; + +export interface Dataset> { + readonly name: string; + fetch(client: PrismaClient): Promise; +} + +interface UserRow extends Record { + id: string; + name: string; + email: string; + created_at: string; +} + +interface MandatoryQuizOutcomeRow extends Record { + user_id: string; + learning_unit_id: string; + learning_unit_title: string; + due_date: string | null; + is_completed: boolean; + is_quiz_attempted: boolean; + is_quiz_passed: boolean | null; + number_of_attempts: number; + updated_at: string; +} + +export const usersDataset: Dataset = { + name: 'users', + async fetch(client) { + const rows = await client.user.findMany({ + select: { id: true, name: true, email: true, createdAt: true }, + }); + return rows.map((r) => ({ + id: r.id, + name: r.name, + email: r.email, + created_at: r.createdAt.toISOString(), + })); + }, +}; + +export const mandatoryQuizOutcomesDataset: Dataset = { + name: 'mandatory_quiz_outcomes', + async fetch(client) { + const rows = await client.learningJourney.findMany({ + select: { + userId: true, + learningUnitId: true, + isCompleted: true, + isQuizAttempted: true, + isQuizPassed: true, + numberOfAttempts: true, + updatedAt: true, + learningUnit: { select: { title: true, dueDate: true } }, + }, + where: { learningUnit: { isRequired: true } }, + }); + return rows.map((r) => ({ + user_id: r.userId, + learning_unit_id: r.learningUnitId, + learning_unit_title: r.learningUnit.title, + due_date: r.learningUnit.dueDate ? r.learningUnit.dueDate.toISOString().slice(0, 10) : null, + is_completed: r.isCompleted, + is_quiz_attempted: r.isQuizAttempted, + is_quiz_passed: r.isQuizPassed, + number_of_attempts: r.numberOfAttempts, + updated_at: r.updatedAt.toISOString(), + })); + }, +}; + +export const datasets: ReadonlyArray>> = [ + usersDataset, + mandatoryQuizOutcomesDataset, +]; +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `pnpm test run scripts/export-nlds.test.ts -t Dataset` +Expected: PASS (3 tests). + +- [ ] **Step 5: Commit** + +```bash +git add scripts/export-nlds.ts scripts/export-nlds.test.ts +git commit -m "feat(export): define users and mandatory quiz outcomes datasets + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +### Task 4: S3 uploader with mandatory SSE-KMS + +**Files:** + +- Modify: `scripts/export-nlds.ts` +- Test: `scripts/export-nlds.test.ts` + +**Interfaces:** + +- Consumes: `ExportConfig` (Task 1); `S3Client`, `PutObjectCommand` from `@aws-sdk/client-s3`. +- Produces: `putObject(client: S3Client, cfg: ExportConfig, key: string, body: string): Promise` + +- [ ] **Step 1: Write the failing test** + +Append to `scripts/export-nlds.test.ts` (add `putObject` to the `./export-nlds.ts` import): + +```ts +describe('putObject', () => { + test('sends the object with SSE-KMS, correct bucket/key and NDJSON content type', async () => { + const send = vi.fn().mockResolvedValue({}); + const client = { send } as unknown as import('@aws-sdk/client-s3').S3Client; + const cfg = { + postgresUrl: 'x', + bucket: 'nlds-bucket', + prefix: 'glow/', + kmsKeyId: 'arn:kms:key', + region: 'ap-southeast-1', + }; + + await putObject(client, cfg, 'glow/users/2026-07-01.ndjson', '{"a":1}\n'); + + expect(send).toHaveBeenCalledTimes(1); + const command = send.mock.calls[0][0]; + expect(command.input).toEqual({ + Bucket: 'nlds-bucket', + Key: 'glow/users/2026-07-01.ndjson', + Body: '{"a":1}\n', + ContentType: 'application/x-ndjson', + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: 'arn:kms:key', + }); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `pnpm test run scripts/export-nlds.test.ts -t putObject` +Expected: FAIL — `putObject` not exported. + +- [ ] **Step 3: Implement `putObject`** + +Add the import at the top of `scripts/export-nlds.ts` and the function: + +```ts +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +``` + +```ts +export async function putObject( + client: S3Client, + cfg: ExportConfig, + key: string, + body: string, +): Promise { + await client.send( + new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Body: body, + ContentType: 'application/x-ndjson', + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: cfg.kmsKeyId, + }), + ); +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `pnpm test run scripts/export-nlds.test.ts -t putObject` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add scripts/export-nlds.ts scripts/export-nlds.test.ts +git commit -m "feat(export): add SSE-KMS S3 uploader + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +### Task 5: Orchestrator (`main`) + entry wrapper + client wiring + +**Files:** + +- Modify: `scripts/export-nlds.ts` +- Test: `scripts/export-nlds.test.ts` + +**Interfaces:** + +- Consumes: everything above; `PrismaClient`, `PrismaPg`. +- Produces: `runExport(deps: { client: PrismaClient; s3: S3Client; cfg: ExportConfig; now: Date }): Promise` where `interface DatasetResult { name: string; key: string; rowCount: number; }`. `runExport` is the testable core; `main()` builds real clients from env and calls it; a bottom-of-file guard runs `main()` only when executed as the entrypoint. + +- [ ] **Step 1: Write the failing tests** + +Append to `scripts/export-nlds.test.ts` (add `runExport` to the `./export-nlds.ts` import): + +```ts +describe('runExport', () => { + const cfg = { + postgresUrl: 'x', + bucket: 'nlds-bucket', + prefix: 'glow/', + kmsKeyId: 'arn:kms:key', + region: 'ap-southeast-1', + }; + + test('uploads both datasets under dated keys and returns their row counts', async () => { + const client = { + user: { + findMany: vi + .fn() + .mockResolvedValue([ + { id: 'u1', name: 'Ada', email: 'a@x', createdAt: new Date('2026-01-01T00:00:00Z') }, + ]), + }, + learningJourney: { findMany: vi.fn().mockResolvedValue([]) }, + } as unknown as import('../src/generated/prisma/client.js').PrismaClient; + const send = vi.fn().mockResolvedValue({}); + const s3 = { send } as unknown as import('@aws-sdk/client-s3').S3Client; + + const results = await runExport({ client, s3, cfg, now: new Date('2026-07-01T00:00:00Z') }); + + expect(results).toEqual([ + { name: 'users', key: 'glow/users/2026-07-01.ndjson', rowCount: 1 }, + { + name: 'mandatory_quiz_outcomes', + key: 'glow/mandatory_quiz_outcomes/2026-07-01.ndjson', + rowCount: 0, + }, + ]); + expect(send).toHaveBeenCalledTimes(2); + }); + + test('propagates an upload failure (aborting the run)', async () => { + const client = { + user: { findMany: vi.fn().mockResolvedValue([]) }, + learningJourney: { findMany: vi.fn().mockResolvedValue([]) }, + } as unknown as import('../src/generated/prisma/client.js').PrismaClient; + const send = vi.fn().mockRejectedValue(new Error('AccessDenied')); + const s3 = { send } as unknown as import('@aws-sdk/client-s3').S3Client; + + await expect( + runExport({ client, s3, cfg, now: new Date('2026-07-01T00:00:00Z') }), + ).rejects.toThrow('AccessDenied'); + }); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `pnpm test run scripts/export-nlds.test.ts -t runExport` +Expected: FAIL — `runExport` not exported. + +- [ ] **Step 3: Implement `runExport`, `main`, and the entry guard** + +Add the imports at the top of `scripts/export-nlds.ts`: + +```ts +import { fileURLToPath } from 'node:url'; +import process from 'node:process'; + +import { PrismaPg } from '@prisma/adapter-pg'; + +import { PrismaClient } from '../src/generated/prisma/client.js'; +``` + +(Change the earlier `import { type PrismaClient }` to a value import as shown, since `main` now constructs one.) + +Add: + +```ts +export interface DatasetResult { + name: string; + key: string; + rowCount: number; +} + +export async function runExport(deps: { + client: PrismaClient; + s3: S3Client; + cfg: ExportConfig; + now: Date; +}): Promise { + const { client, s3, cfg, now } = deps; + const date = formatDateUtc(now); + const results: DatasetResult[] = []; + for (const dataset of datasets) { + const rows = await dataset.fetch(client); + const key = objectKey(cfg.prefix, dataset.name, date); + await putObject(s3, cfg, key, toNdjson(rows)); + results.push({ name: dataset.name, key, rowCount: rows.length }); + } + return results; +} + +async function main(): Promise { + const cfg = loadConfig(process.env); + const client = new PrismaClient({ + adapter: new PrismaPg({ connectionString: cfg.postgresUrl }), + }); + const s3 = new S3Client({ region: cfg.region }); + try { + const results = await runExport({ client, s3, cfg, now: new Date() }); + for (const r of results) { + console.log(`exported ${r.rowCount} rows -> s3://${cfg.bucket}/${r.key}`); + } + } finally { + await client.$disconnect(); + } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error('NLDS export failed:', err); + process.exit(1); + }); +} +``` + +- [ ] **Step 4: Run the full test file to verify everything passes** + +Run: `pnpm test run scripts/export-nlds.test.ts` +Expected: PASS (all tests across Tasks 1–5). + +- [ ] **Step 5: Type-check** + +Run: `pnpm check` +Expected: no errors. (If `scripts/` is not covered by `tsconfig`, the check still validates the file via `allowJs`/`checkJs` include; fix any reported type errors before committing.) + +- [ ] **Step 6: Commit** + +```bash +git add scripts/export-nlds.ts scripts/export-nlds.test.ts +git commit -m "feat(export): add export orchestrator and entrypoint + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +### Task 6: Packaging — npm scripts, esbuild devDep, Dockerfile bundling + +Wires the fixed `pnpm run export:nlds` command and the build-time bundling per ADR-0006. Ends with a real bundle + local run against the dev DB (the acceptance check). + +**Files:** + +- Modify: `package.json` +- Modify: `Dockerfile:42-46` + +**Interfaces:** + +- Consumes: `scripts/export-nlds.ts` (Task 5). +- Produces: `build/export-nlds.js` (bundled), `pnpm run export:nlds` entrypoint. + +- [ ] **Step 1: Add esbuild as a devDependency** + +Run: `pnpm add -D esbuild` +Expected: `esbuild` appears under `devDependencies` in `package.json` and `pnpm-lock.yaml` updates. + +- [ ] **Step 2: Add the npm scripts** + +Edit `package.json` `scripts` — add these two entries: + +```json + "export:nlds:bundle": "esbuild scripts/export-nlds.ts --bundle --platform=node --format=esm --target=node24 --packages=external --outfile=build/export-nlds.js", + "export:nlds": "node build/export-nlds.js", +``` + +Note: `--packages=external` keeps `@aws-sdk/client-s3`, `@prisma/adapter-pg`, `@prisma/client`, and `pg` external (resolved from prod `node_modules`); only `scripts/export-nlds.ts` and the relatively-imported generated Prisma client are inlined. + +- [ ] **Step 3: Verify the bundle builds and runs end-to-end (acceptance)** + +Start local infra if needed (`docker compose up -d` for Postgres + MinIO per README), seed if needed (`pnpm db:seed`), then: + +```bash +pnpm export:nlds:bundle +NLDS_BUCKET=onward NLDS_PREFIX=test/ NLDS_KMS_KEY_ID= AWS_REGION=ap-southeast-1 \ + node --env-file=.env build/export-nlds.js +``` + +Expected: two `exported N rows -> s3://onward/test/...ndjson` log lines and exit code `0` (`echo $?` → `0`). + +If your local S3 is MinIO (no KMS), point at the real dev bucket/prefix and KMS key instead, or accept that the SSE-KMS assertion requires a KMS-capable target — the acceptance is that a KMS-capable target produces the objects and exits `0`. + +- [ ] **Step 4: Verify the failure path exits non-zero** + +Run (omit a required var): + +```bash +NLDS_BUCKET=onward NLDS_PREFIX=test/ AWS_REGION=ap-southeast-1 node --env-file=.env build/export-nlds.js; echo $? +``` + +Expected: logs `NLDS export failed: ... NLDS_KMS_KEY_ID` and prints a non-zero exit code. + +- [ ] **Step 5: Add the Dockerfile bundling step** + +In `Dockerfile`, in the build stage, insert the bundle step after `pnpm build` (line 42) and **before** the production-dependency prune (lines 45–46), so `esbuild` (a devDep) is still installed: + +```dockerfile +# Build the app. +RUN pnpm build + +# Bundle the NLDS export script into build/ (see ADR-0006). +RUN pnpm export:nlds:bundle + +# Install production dependencies. +RUN find . -type d -name "node_modules" -prune -exec rm -rf {} + +RUN pnpm install --offline --prod +``` + +`build/export-nlds.js` is carried into the production stage by the existing `COPY --from=build … /app/build ./build`; no new `COPY` line is needed. + +- [ ] **Step 6: Verify the image build produces and runs the bundle** + +Run: + +```bash +docker build -t onward-nlds-test . +docker run --rm --entrypoint node onward-nlds-test build/export-nlds.js; echo $? +``` + +Expected: the script starts and fails on missing env (non-zero) — proving the bundle exists and is runnable in the prod image with only prod deps present. (A full success run requires DB + bucket env wired into the container.) + +- [ ] **Step 7: Run the full test + type check once more** + +Run: `pnpm test run scripts/export-nlds.test.ts && pnpm check` +Expected: PASS, no type errors. + +- [ ] **Step 8: Commit** + +```bash +git add package.json pnpm-lock.yaml Dockerfile +git commit -m "chore(export): wire export:nlds command and build-time bundling + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Notes / Risks + +- **Prisma 7 client bundling:** the generated client is inlined; its runtime library (`@prisma/client`) and the query compiler stay **external** (present in prod `node_modules`). If the image run in Task 6 Step 6 fails to load a runtime/wasm asset, the fix is to keep more of `@prisma/*` external (already the default via `--packages=external`) — do not inline the Prisma runtime. Task 6 Step 6 is the gate that catches this before merge. +- **Memory:** each dataset is buffered fully in memory before upload. Acceptable at current table sizes; if volume grows, switch `putObject` to a streamed multipart upload — an internal change that does not alter its signature. +- **`.ts` extension in imports:** this repo's test imports use `./file.js` for runtime modules; the plan's test imports use `./export-nlds.ts` / `../src/generated/prisma/client.js` to match the actual on-disk extension under Vite's resolver. If `pnpm check`/`pnpm test` complains about the extension, align it with the repo's prevailing convention (`.js`) — verify against a neighboring passing test. + +``` + +``` diff --git a/docs/superpowers/specs/2026-07-01-nlds-export-design.md b/docs/superpowers/specs/2026-07-01-nlds-export-design.md new file mode 100644 index 00000000..f9bc26c3 --- /dev/null +++ b/docs/superpowers/specs/2026-07-01-nlds-export-design.md @@ -0,0 +1,353 @@ +# NLDS Mandatory-Quiz-Completion Export — Design + +**Status:** Draft +**Depends on / Related:** [ADR-0006](../../decisions/0006-package-nlds-export-as-bundled-script-in-app-image.md) (packaging), infra spec `export:nlds` (invocation contract) + +## Overview + +A daily scheduled ECS task must export curated data from the Onward database to +the NLDS-owned S3 data lake so NLDS can track **mandatory quiz completion** — +which users have (and have not) completed the mandatory learning modules. The +export is a batch script that runs to completion and exits: it connects to the +database read-only, produces one object per dataset, uploads them to the +cross-account bucket under a scoped prefix with SSE-KMS, and exits `0` on full +success or non-zero on any failure (the scheduler alerts on non-zero). + +This spec covers **two datasets only**: `users` (identity dimension) and +`mandatory_quiz_outcomes` (the per-user completion fact for mandatory modules). +Each run writes a **full snapshot** (current state of all rows), not an +incremental delta, because completion tracking needs a complete point-in-time +picture of who is outstanding — a single day's file must answer that on its own. + +Packaging and execution (how the script runs inside the app image with only +Prisma + the S3 client available) are decided in +[ADR-0006](../../decisions/0006-package-nlds-export-as-bundled-script-in-app-image.md); +this spec consumes that outcome. **Out of scope** (per the infra ADR): dataset +selection rationale beyond the two above, warehouse schema mapping, historical +backfill, and the NLDS-side bucket/ingestion. + +## Goals + +- **Auditability / completeness:** each daily run emits a complete snapshot from + which "who has not completed which mandatory module" is answerable from one + file. +- **Fail-loud:** any query, serialization, config, or upload error aborts the run + with a non-zero exit and a logged cause; no partial-success masquerading as + success. +- **Least data leaving Glow:** only the explicitly enumerated columns are + exported — no `SELECT *`, no fields beyond those listed here. +- **No new runtime dependencies:** uses only Prisma and `@aws-sdk/client-s3` + (per ADR-0006); the bundler is build-stage only. +- **Idempotent per day:** re-running overwrites the same day's key; never + appends. + +## Requirements + +Invocation contract (fixed by infra — not changed here): + +| Aspect | Value | +| --------- | -------------------------------------------- | +| Command | `pnpm run export:nlds` | +| Exit code | `0` on full success; non-zero on any failure | +| Runtime | Node, ARM64, no shell tools (SDK only) | + +Environment (injected by the task definition): + +| Var | Meaning | +| ----------------- | ----------------------------------------------------------------------- | +| `POSTGRES_URL` | DB connection (read-only use) | +| `NLDS_BUCKET` | target bucket name | +| `NLDS_PREFIX` | key prefix the role is scoped to (e.g. `glow/`); never write outside it | +| `NLDS_KMS_KEY_ID` | KMS key ARN for SSE-KMS on every `PutObject` | +| `AWS_REGION` | AWS region (e.g. `ap-southeast-1`) | + +Output objects (NDJSON — one JSON object per line, UTF-8, trailing newline): + +- `${NLDS_PREFIX}users/${YYYY-MM-DD}.ndjson` +- `${NLDS_PREFIX}mandatory_quiz_outcomes/${YYYY-MM-DD}.ndjson` + +`YYYY-MM-DD` is the run date in **UTC**. Keys are relative to `NLDS_PREFIX`; the +script must never construct a key outside it. + +`users` record fields (identity dimension — **contains PII**): + +| Field | Source | Type | +| ------------ | ------------------ | ----------------- | +| `id` | `users.id` | string (uuid) | +| `name` | `users.name` | string | +| `email` | `users.email` | string | +| `created_at` | `users.created_at` | string (ISO 8601) | + +`mandatory_quiz_outcomes` record fields (completion fact): + +| Field | Source | Type | +| --------------------- | -------------------------------------- | --------------------- | +| `user_id` | `learning_journeys.user_id` | string (uuid) | +| `learning_unit_id` | `learning_journeys.learning_unit_id` | string (uuid) | +| `learning_unit_title` | `learning_units.title` | string | +| `due_date` | `learning_units.due_date` | string (date) \| null | +| `is_completed` | `learning_journeys.is_completed` | boolean | +| `is_quiz_attempted` | `learning_journeys.is_quiz_attempted` | boolean | +| `is_quiz_passed` | `learning_journeys.is_quiz_passed` | boolean \| null | +| `number_of_attempts` | `learning_journeys.number_of_attempts` | number | +| `updated_at` | `learning_journeys.updated_at` | string (ISO 8601) | + +## Data model + +"Mandatory module" = a `LearningUnit` with `is_required = true`. The completion +outcome is the `LearningJourney` row (unique per `user_id` + `learning_unit_id`), +which already carries `is_completed`, `is_quiz_attempted`, `is_quiz_passed`, and +`number_of_attempts`. The `mandatory_quiz_outcomes` dataset is the set of +`LearningJourney` rows whose related `LearningUnit.is_required = true`, projected +to the fields above. + +```mermaid +erDiagram + User ||--o{ LearningJourney : has + LearningUnit ||--o{ LearningJourney : "tracked by" + LearningUnit { + boolean is_required + date due_date + } + LearningJourney { + boolean is_completed + boolean is_quiz_attempted + boolean is_quiz_passed + int number_of_attempts + } +``` + +The `users` dataset is the full `User` table projected to the four identity +fields (no dependency on journeys), so users with zero mandatory-module journeys +are still present as a dimension. + +## Architecture + +A single standalone entry module orchestrates the run. It loads and validates +config from the environment, opens one Prisma client over the pg adapter (mirror +of the seed scripts, not the SvelteKit `$lib/server/db` module, which depends on +Vite-resolved aliases), then processes each dataset in turn: fetch rows via a +scoped Prisma `select`, serialize to NDJSON, and `PutObject` with SSE-KMS. It +logs per-dataset row count and object key, disconnects, and exits with a status +that reflects whether all datasets succeeded. + +Datasets are values implementing a common `Dataset` contract, so adding or +changing a dataset is a local change to its `fetch` and does not touch the +orchestrator. Serialization and upload are dataset-agnostic. + +Per ADR-0006, the module is bundled ahead of time in the Docker build stage into +`build/export-nlds.js` (generated Prisma client inlined; AWS SDK and Prisma +runtime packages left external, resolved from production `node_modules`). +`pnpm run export:nlds` executes that bundle with the bare Node runtime. + +```mermaid +flowchart LR + ENV[process.env] --> CFG[loadConfig] + CFG --> MAIN[main] + MAIN --> DB[(Postgres via Prisma)] + DB --> FETCH[dataset.fetch] + FETCH --> NDJSON[toNdjson] + NDJSON --> PUT[putObject SSE-KMS] + PUT --> S3[(NLDS S3 bucket)] + MAIN -->|all ok| EXIT0[exit 0] + MAIN -->|any error| EXITN[exit non-zero] +``` + +## Contracts & boundaries + +### `ExportConfig` + +- **Does:** validated, typed snapshot of the environment the run needs. +- **Use:** + +```ts +interface ExportConfig { + postgresUrl: string; + bucket: string; + prefix: string; + kmsKeyId: string; + region: string; +} +``` + +- **Guarantees:** every field is a non-empty string; `prefix` is used verbatim as + a key prefix (caller passes it including any trailing `/`). + +### `loadConfig` + +- **Does:** reads the required env vars and builds an `ExportConfig`. +- **Use:** `loadConfig(env: NodeJS.ProcessEnv): ExportConfig;` +- **Depends on:** the environment table above. +- **Guarantees:** returns a fully-populated `ExportConfig`. +- **Requires:** throws a descriptive error naming the first missing/empty + variable — no defaults, no partial config. + +### `Dataset` + +- **Does:** names one exported dataset and knows how to fetch its rows. +- **Use:** + +```ts +interface Dataset> { + readonly name: string; + fetch(client: PrismaClient): Promise; +} +``` + +- **Guarantees:** `name` is the key path segment (e.g. `users`); `fetch` + performs a read-only, explicit-column `select` (never `SELECT *`) and returns + plain serializable rows. +- **Requires:** a connected `PrismaClient`. + +### dataset values `usersDataset`, `mandatoryQuizOutcomesDataset` + +- **Does:** the two concrete datasets from the Requirements tables. +- **Use:** `const datasets: ReadonlyArray>>;` +- **Guarantees:** `usersDataset` projects the four identity fields; + `mandatoryQuizOutcomesDataset` returns journeys where + `learningUnit.is_required = true`, projected to the nine fields; dates are + emitted as ISO-8601 strings and `null`s preserved (not coerced to empty). + +### `objectKey` + +- **Does:** builds the S3 key for a dataset on a date. +- **Use:** `objectKey(prefix: string, dataset: string, date: string): string;` +- **Guarantees:** returns `` `${prefix}${dataset}/${date}.ndjson` `` and nothing + that escapes `prefix`. + +### `formatDateUtc` + +- **Does:** formats a timestamp as the run's date key. +- **Use:** `formatDateUtc(now: Date): string;` +- **Guarantees:** `YYYY-MM-DD` in UTC. + +### `toNdjson` + +- **Does:** serializes rows to NDJSON. +- **Use:** `toNdjson(rows: ReadonlyArray>): string;` +- **Guarantees:** one `JSON.stringify` per line, `\n`-terminated including the + last line; empty input yields `''`. + +### `putObject` + +- **Does:** uploads one object with mandatory SSE-KMS. +- **Use:** + `putObject(client: S3Client, cfg: ExportConfig, key: string, body: string): Promise;` +- **Depends on:** `@aws-sdk/client-s3` `PutObjectCommand`. +- **Guarantees:** sends `Bucket = cfg.bucket`, `Key = key`, `Body = body`, + `ContentType = 'application/x-ndjson'`, `ServerSideEncryption = 'aws:kms'`, + `SSEKMSKeyId = cfg.kmsKeyId`. Rejects if the put fails. +- **Requires:** `key` starts with `cfg.prefix`. + +### `main` + +- **Does:** orchestrates the whole run. +- **Use:** `main(): Promise;` (module entry; process exit code set by the + wrapper around it). +- **Guarantees:** processes every dataset; logs `{ dataset, key, rowCount }` per + success; disconnects the client in all paths. Resolves only if **all** datasets + uploaded; otherwise the process exits non-zero (see Error handling). + +## Components / changes + +### 1. `scripts/export-nlds.ts` (new) + +The entry module implementing every contract above. Instantiates its own +`PrismaClient` over `@prisma/adapter-pg` using `cfg.postgresUrl` (pattern from +`prisma/seed.ts`), and a default `new S3Client()` (task-role credentials + +`AWS_REGION` from the environment). Bottom-of-file wrapper runs `main()`, logs +the failing dataset/cause and calls `process.exit(1)` on rejection, and +`$disconnect()`s in `finally`. No `dotenv` import (env is injected in prod; local +runs use Node's `--env-file`). + +### 2. `package.json` (modified) + +- Add `"export:nlds": "node build/export-nlds.js"` — the fixed prod entrypoint. +- Add `"export:nlds:bundle": "esbuild scripts/export-nlds.ts --bundle --platform=node --format=esm --target=node24 --outfile=build/export-nlds.js"` — used by the Docker build and by local runs. +- Add `esbuild` to **devDependencies** (build-stage only; never shipped to the + runtime stage — see ADR-0006). + +### 3. `Dockerfile` (modified, build stage) + +After `pnpm build` and before the production-dependency prune, add +`RUN pnpm export:nlds:bundle` so `build/export-nlds.js` exists. It rides the +existing `COPY --from=build … /app/build ./build` into the production stage; no +new `COPY` and no runtime-stage change. + +## Error handling + +- **Missing/empty env (config):** `loadConfig` throws before any DB or S3 work; + the wrapper logs and exits non-zero. +- **DB connect / query failure:** the connect check or a `fetch` rejects; the run + aborts, the wrapper logs the dataset and cause, and exits non-zero. Access is + read-only (`select` only) — the source DB is never written. +- **Serialization failure:** a non-serializable value throwing in `toNdjson` + aborts that dataset and the run. +- **Upload failure (incl. SSE-KMS/prefix denials):** a rejected `PutObject` + (e.g. the bucket rejecting a put without SSE-KMS, or a key outside the granted + prefix) aborts the run non-zero. Because datasets are processed in sequence and + the first failure aborts, a run either completes all datasets or exits + non-zero — there is no silent partial success. Objects already uploaded before + a later failure remain (idempotent overwrite fixes them on the next run); this + is acceptable and surfaced by the non-zero exit + alert. +- **Client cleanup:** `$disconnect()` runs in `finally` on every path. + +## Security considerations + +- **Access control (OWASP A01; STRIDE: EoP).** The task role is scoped by NLDS's + bucket + key policy to `${NLDS_PREFIX}*`; the script never assumes a role and + never builds a key outside `cfg.prefix` (`objectKey` enforces the prefix). DB + access uses the injected read-only-intent URL and issues only `select`s. +- **Encryption / information disclosure (STRIDE: Info Disclosure).** Every + `PutObject` sets `ServerSideEncryption: 'aws:kms'` + `SSEKMSKeyId`; the bucket + rejects non-KMS puts, so a regression fails loudly rather than writing + plaintext. Data in transit uses the SDK's HTTPS default. +- **PII (OWASP A01/A02).** The `users` dataset exports `name` + `email` — PII + leaving Glow. Mitigation: the export is limited to exactly the enumerated + identity columns (no `SELECT *`), lands only in the KMS-encrypted, + policy-scoped NLDS prefix, and the PII scope is called out here for review. No + credentials, tokens, or chat/message content are exported. +- **Injection (OWASP A03).** N/A for query construction — all queries go through + Prisma's parameterized `select`; no string-built SQL. Keys are built from a + fixed prefix + fixed dataset name + a formatted date, not user input. +- **Secrets handling.** `POSTGRES_URL` and KMS key ARN come from the environment + (Secrets Manager / task env); none are logged. Logs contain only dataset name, + object key, and row count. +- **Denial of Service (STRIDE).** N/A — single scheduled batch, no inbound + surface. Row volume is bounded by table size; see Testing note on memory. + +## Testing + +Unit-level (pure functions, no I/O): + +- `loadConfig` — returns a full config when all vars present; throws naming the + first missing var (one case per required var, or a representative subset). +- `objectKey` — composes `${prefix}${dataset}/${date}.ndjson`; a dataset name + never produces a key outside the prefix. +- `formatDateUtc` — a known timestamp maps to the expected UTC `YYYY-MM-DD`, + including a time near a UTC day boundary. +- `toNdjson` — multiple rows → one `\n`-terminated JSON line each; empty array → + `''`; a row with a `null` field preserves `null` (not `""`). + +Dataset/upload behavior (Prisma + S3 client mocked): + +- `mandatoryQuizOutcomesDataset.fetch` — issues a `select` filtered to + `learningUnit.is_required = true` and projects exactly the nine fields; a + journey on a non-required unit is excluded. +- `usersDataset.fetch` — projects exactly the four identity fields; a user with + no journeys still appears. +- `putObject` — the `PutObjectCommand` input carries + `ServerSideEncryption: 'aws:kms'` and `SSEKMSKeyId` from config, correct + bucket/key, and `application/x-ndjson` content type. +- `main` — happy path uploads both datasets and resolves; a rejected `fetch` or + `putObject` propagates so the wrapper exits non-zero; `$disconnect` is called on + both success and failure paths. + +Acceptance (per infra "Done when"): `pnpm run export:nlds` locally (dev DB + +test prefix, real or MinIO S3) produces both `.ndjson` objects and exits `0`; a +forced failure (e.g. missing `NLDS_KMS_KEY_ID`) exits non-zero. + +Note (memory): a full snapshot is buffered in memory per dataset before upload. +This is acceptable at current table sizes; if `users` or journeys grow large, +switch `putObject` to a streamed multipart upload — a `putObject`-internal change +that does not affect its contract. From ce91d2aeff5fd4a030933f1a1f580002338448cb Mon Sep 17 00:00:00 2001 From: Nicholas Lim Date: Wed, 1 Jul 2026 17:41:33 +0800 Subject: [PATCH 2/8] feat(export): add NLDS export config loader and key helpers Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/export-nlds.test.ts | 49 +++++++++++++++++++++++++++++++++++++ scripts/export-nlds.ts | 40 ++++++++++++++++++++++++++++++ vitest.config.ts | 2 +- 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 scripts/export-nlds.test.ts create mode 100644 scripts/export-nlds.ts diff --git a/scripts/export-nlds.test.ts b/scripts/export-nlds.test.ts new file mode 100644 index 00000000..e64bbdaa --- /dev/null +++ b/scripts/export-nlds.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from 'vitest'; + +import { formatDateUtc, loadConfig, objectKey } from './export-nlds.js'; + +const fullEnv = { + POSTGRES_URL: 'postgresql://u:p@localhost:5432/db', + NLDS_BUCKET: 'nlds-bucket', + NLDS_PREFIX: 'glow/', + NLDS_KMS_KEY_ID: 'arn:aws:kms:ap-southeast-1:1:key/abc', + AWS_REGION: 'ap-southeast-1', +} satisfies NodeJS.ProcessEnv; + +describe('loadConfig', () => { + test('returns a fully-populated config when all vars are present', () => { + expect(loadConfig(fullEnv)).toEqual({ + postgresUrl: 'postgresql://u:p@localhost:5432/db', + bucket: 'nlds-bucket', + prefix: 'glow/', + kmsKeyId: 'arn:aws:kms:ap-southeast-1:1:key/abc', + region: 'ap-southeast-1', + }); + }); + + test('throws naming the missing var', () => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { NLDS_KMS_KEY_ID, ...missing } = fullEnv; + expect(() => loadConfig(missing)).toThrow('NLDS_KMS_KEY_ID'); + }); + + test('treats an empty-string var as missing', () => { + expect(() => loadConfig({ ...fullEnv, NLDS_BUCKET: '' })).toThrow('NLDS_BUCKET'); + }); +}); + +describe('objectKey', () => { + test('composes prefix + dataset + date + .ndjson', () => { + expect(objectKey('glow/', 'users', '2026-07-01')).toBe('glow/users/2026-07-01.ndjson'); + }); +}); + +describe('formatDateUtc', () => { + test('formats as UTC YYYY-MM-DD', () => { + expect(formatDateUtc(new Date('2026-07-01T13:30:00Z'))).toBe('2026-07-01'); + }); + + test('uses UTC, not local time, near a day boundary', () => { + expect(formatDateUtc(new Date('2026-07-01T23:30:00Z'))).toBe('2026-07-01'); + }); +}); diff --git a/scripts/export-nlds.ts b/scripts/export-nlds.ts new file mode 100644 index 00000000..ee0cd03a --- /dev/null +++ b/scripts/export-nlds.ts @@ -0,0 +1,40 @@ +export interface ExportConfig { + postgresUrl: string; + bucket: string; + prefix: string; + kmsKeyId: string; + region: string; +} + +type RequiredVar = + | 'POSTGRES_URL' + | 'NLDS_BUCKET' + | 'NLDS_PREFIX' + | 'NLDS_KMS_KEY_ID' + | 'AWS_REGION'; + +function requireVar(env: NodeJS.ProcessEnv, name: RequiredVar): string { + const value = env[name]; + if (!value) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +} + +export function loadConfig(env: NodeJS.ProcessEnv): ExportConfig { + return { + postgresUrl: requireVar(env, 'POSTGRES_URL'), + bucket: requireVar(env, 'NLDS_BUCKET'), + prefix: requireVar(env, 'NLDS_PREFIX'), + kmsKeyId: requireVar(env, 'NLDS_KMS_KEY_ID'), + region: requireVar(env, 'AWS_REGION'), + }; +} + +export function objectKey(prefix: string, dataset: string, date: string): string { + return `${prefix}${dataset}/${date}.ndjson`; +} + +export function formatDateUtc(now: Date): string { + return now.toISOString().slice(0, 10); +} diff --git a/vitest.config.ts b/vitest.config.ts index 322a0cc0..a6d7a262 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -6,7 +6,7 @@ export default { test: { globals: true, environment: 'jsdom', - include: ['src/**/*.{test,spec}.{js,ts}'], + include: ['src/**/*.{test,spec}.{js,ts}', 'scripts/**/*.{test,spec}.{js,ts}'], setupFiles: ['./vitest.setup.ts'], coverage: { enabled: true, From 167211d59595959bb78f2c2dd89e61d8dd1c70c5 Mon Sep 17 00:00:00 2001 From: Nicholas Lim Date: Wed, 1 Jul 2026 17:46:11 +0800 Subject: [PATCH 3/8] feat(export): add NDJSON serializer Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/export-nlds.test.ts | 17 ++++++++++++++++- scripts/export-nlds.ts | 4 ++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/scripts/export-nlds.test.ts b/scripts/export-nlds.test.ts index e64bbdaa..b13e7e25 100644 --- a/scripts/export-nlds.test.ts +++ b/scripts/export-nlds.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { formatDateUtc, loadConfig, objectKey } from './export-nlds.js'; +import { formatDateUtc, loadConfig, objectKey, toNdjson } from './export-nlds.js'; const fullEnv = { POSTGRES_URL: 'postgresql://u:p@localhost:5432/db', @@ -47,3 +47,18 @@ describe('formatDateUtc', () => { expect(formatDateUtc(new Date('2026-07-01T23:30:00Z'))).toBe('2026-07-01'); }); }); + +describe('toNdjson', () => { + test('serializes each row as its own newline-terminated JSON line', () => { + const out = toNdjson([{ a: 1 }, { a: 2 }]); + expect(out).toBe('{"a":1}\n{"a":2}\n'); + }); + + test('returns empty string for no rows', () => { + expect(toNdjson([])).toBe(''); + }); + + test('preserves null fields (does not coerce to empty string)', () => { + expect(toNdjson([{ a: null }])).toBe('{"a":null}\n'); + }); +}); diff --git a/scripts/export-nlds.ts b/scripts/export-nlds.ts index ee0cd03a..aa2f12d9 100644 --- a/scripts/export-nlds.ts +++ b/scripts/export-nlds.ts @@ -38,3 +38,7 @@ export function objectKey(prefix: string, dataset: string, date: string): string export function formatDateUtc(now: Date): string { return now.toISOString().slice(0, 10); } + +export function toNdjson(rows: readonly Record[]): string { + return rows.map((row) => `${JSON.stringify(row)}\n`).join(''); +} From d6b5bbd7561fc8a2050fa3c6006bd7b5f174628d Mon Sep 17 00:00:00 2001 From: Nicholas Lim Date: Wed, 1 Jul 2026 17:49:32 +0800 Subject: [PATCH 4/8] feat(export): define users and mandatory quiz outcomes datasets Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/export-nlds.test.ts | 104 +++++++++++++++++++++++++++++++++++- scripts/export-nlds.ts | 76 ++++++++++++++++++++++++++ 2 files changed, 178 insertions(+), 2 deletions(-) diff --git a/scripts/export-nlds.test.ts b/scripts/export-nlds.test.ts index b13e7e25..aa754060 100644 --- a/scripts/export-nlds.test.ts +++ b/scripts/export-nlds.test.ts @@ -1,6 +1,13 @@ -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; -import { formatDateUtc, loadConfig, objectKey, toNdjson } from './export-nlds.js'; +import { + formatDateUtc, + loadConfig, + mandatoryQuizOutcomesDataset, + objectKey, + toNdjson, + usersDataset, +} from './export-nlds.js'; const fullEnv = { POSTGRES_URL: 'postgresql://u:p@localhost:5432/db', @@ -62,3 +69,96 @@ describe('toNdjson', () => { expect(toNdjson([{ a: null }])).toBe('{"a":null}\n'); }); }); + +describe('usersDataset', () => { + test('selects exactly the four identity fields and maps to snake_case rows', async () => { + const findMany = vi + .fn() + .mockResolvedValue([ + { id: 'u1', name: 'Ada', email: 'ada@x.gov', createdAt: new Date('2026-01-02T00:00:00Z') }, + ]); + const client = { + user: { findMany }, + } as unknown as import('../src/generated/prisma/client.js').PrismaClient; + + const rows = await usersDataset.fetch(client); + + expect(findMany).toHaveBeenCalledWith({ + select: { id: true, name: true, email: true, createdAt: true }, + }); + expect(rows).toEqual([ + { id: 'u1', name: 'Ada', email: 'ada@x.gov', created_at: '2026-01-02T00:00:00.000Z' }, + ]); + }); +}); + +describe('mandatoryQuizOutcomesDataset', () => { + test('filters to required units and projects the nine fields as snake_case', async () => { + const findMany = vi.fn().mockResolvedValue([ + { + userId: 'u1', + learningUnitId: 'lu1', + isCompleted: true, + isQuizAttempted: true, + isQuizPassed: null, + numberOfAttempts: 2, + updatedAt: new Date('2026-06-30T10:00:00Z'), + learningUnit: { title: 'Cyber Hygiene', dueDate: new Date('2026-12-31T00:00:00Z') }, + }, + ]); + const client = { + learningJourney: { findMany }, + } as unknown as import('../src/generated/prisma/client.js').PrismaClient; + + const rows = await mandatoryQuizOutcomesDataset.fetch(client); + + expect(findMany).toHaveBeenCalledWith({ + select: { + userId: true, + learningUnitId: true, + isCompleted: true, + isQuizAttempted: true, + isQuizPassed: true, + numberOfAttempts: true, + updatedAt: true, + learningUnit: { select: { title: true, dueDate: true } }, + }, + where: { learningUnit: { isRequired: true } }, + }); + expect(rows).toEqual([ + { + user_id: 'u1', + learning_unit_id: 'lu1', + learning_unit_title: 'Cyber Hygiene', + due_date: '2026-12-31', + is_completed: true, + is_quiz_attempted: true, + is_quiz_passed: null, + number_of_attempts: 2, + updated_at: '2026-06-30T10:00:00.000Z', + }, + ]); + }); + + test('emits null due_date when the unit has none', async () => { + const findMany = vi.fn().mockResolvedValue([ + { + userId: 'u1', + learningUnitId: 'lu1', + isCompleted: false, + isQuizAttempted: false, + isQuizPassed: null, + numberOfAttempts: 0, + updatedAt: new Date('2026-06-30T10:00:00Z'), + learningUnit: { title: 'AI Literacy', dueDate: null }, + }, + ]); + const client = { + learningJourney: { findMany }, + } as unknown as import('../src/generated/prisma/client.js').PrismaClient; + + const [row] = await mandatoryQuizOutcomesDataset.fetch(client); + + expect(row.due_date).toBeNull(); + }); +}); diff --git a/scripts/export-nlds.ts b/scripts/export-nlds.ts index aa2f12d9..3caf4f5a 100644 --- a/scripts/export-nlds.ts +++ b/scripts/export-nlds.ts @@ -1,3 +1,5 @@ +import { type PrismaClient } from '../src/generated/prisma/client.js'; + export interface ExportConfig { postgresUrl: string; bucket: string; @@ -42,3 +44,77 @@ export function formatDateUtc(now: Date): string { export function toNdjson(rows: readonly Record[]): string { return rows.map((row) => `${JSON.stringify(row)}\n`).join(''); } + +export interface Dataset> { + readonly name: string; + fetch(client: PrismaClient): Promise; +} + +interface UserRow extends Record { + id: string; + name: string; + email: string; + created_at: string; +} + +interface MandatoryQuizOutcomeRow extends Record { + user_id: string; + learning_unit_id: string; + learning_unit_title: string; + due_date: string | null; + is_completed: boolean; + is_quiz_attempted: boolean; + is_quiz_passed: boolean | null; + number_of_attempts: number; + updated_at: string; +} + +export const usersDataset: Dataset = { + name: 'users', + async fetch(client) { + const rows = await client.user.findMany({ + select: { id: true, name: true, email: true, createdAt: true }, + }); + return rows.map((r) => ({ + id: r.id, + name: r.name, + email: r.email, + created_at: r.createdAt.toISOString(), + })); + }, +}; + +export const mandatoryQuizOutcomesDataset: Dataset = { + name: 'mandatory_quiz_outcomes', + async fetch(client) { + const rows = await client.learningJourney.findMany({ + select: { + userId: true, + learningUnitId: true, + isCompleted: true, + isQuizAttempted: true, + isQuizPassed: true, + numberOfAttempts: true, + updatedAt: true, + learningUnit: { select: { title: true, dueDate: true } }, + }, + where: { learningUnit: { isRequired: true } }, + }); + return rows.map((r) => ({ + user_id: r.userId, + learning_unit_id: r.learningUnitId, + learning_unit_title: r.learningUnit.title, + due_date: r.learningUnit.dueDate ? r.learningUnit.dueDate.toISOString().slice(0, 10) : null, + is_completed: r.isCompleted, + is_quiz_attempted: r.isQuizAttempted, + is_quiz_passed: r.isQuizPassed, + number_of_attempts: r.numberOfAttempts, + updated_at: r.updatedAt.toISOString(), + })); + }, +}; + +export const datasets: readonly Dataset>[] = [ + usersDataset, + mandatoryQuizOutcomesDataset, +]; From df1465f226cf0707c1b47ba8c80957d7e9c13258 Mon Sep 17 00:00:00 2001 From: Nicholas Lim Date: Wed, 1 Jul 2026 17:55:41 +0800 Subject: [PATCH 5/8] feat(export): add SSE-KMS S3 uploader Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/export-nlds.test.ts | 28 ++++++++++++++++++++++++++++ scripts/export-nlds.ts | 20 ++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/scripts/export-nlds.test.ts b/scripts/export-nlds.test.ts index aa754060..13ddbb41 100644 --- a/scripts/export-nlds.test.ts +++ b/scripts/export-nlds.test.ts @@ -5,6 +5,7 @@ import { loadConfig, mandatoryQuizOutcomesDataset, objectKey, + putObject, toNdjson, usersDataset, } from './export-nlds.js'; @@ -162,3 +163,30 @@ describe('mandatoryQuizOutcomesDataset', () => { expect(row.due_date).toBeNull(); }); }); + +describe('putObject', () => { + test('sends the object with SSE-KMS, correct bucket/key and NDJSON content type', async () => { + const send = vi.fn().mockResolvedValue({}); + const client = { send } as unknown as import('@aws-sdk/client-s3').S3Client; + const cfg = { + postgresUrl: 'x', + bucket: 'nlds-bucket', + prefix: 'glow/', + kmsKeyId: 'arn:kms:key', + region: 'ap-southeast-1', + }; + + await putObject(client, cfg, 'glow/users/2026-07-01.ndjson', '{"a":1}\n'); + + expect(send).toHaveBeenCalledTimes(1); + const command = send.mock.calls[0][0]; + expect(command.input).toEqual({ + Bucket: 'nlds-bucket', + Key: 'glow/users/2026-07-01.ndjson', + Body: '{"a":1}\n', + ContentType: 'application/x-ndjson', + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: 'arn:kms:key', + }); + }); +}); diff --git a/scripts/export-nlds.ts b/scripts/export-nlds.ts index 3caf4f5a..e8664421 100644 --- a/scripts/export-nlds.ts +++ b/scripts/export-nlds.ts @@ -1,3 +1,5 @@ +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; + import { type PrismaClient } from '../src/generated/prisma/client.js'; export interface ExportConfig { @@ -114,6 +116,24 @@ export const mandatoryQuizOutcomesDataset: Dataset = { }, }; +export async function putObject( + client: S3Client, + cfg: ExportConfig, + key: string, + body: string, +): Promise { + await client.send( + new PutObjectCommand({ + Bucket: cfg.bucket, + Key: key, + Body: body, + ContentType: 'application/x-ndjson', + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: cfg.kmsKeyId, + }), + ); +} + export const datasets: readonly Dataset>[] = [ usersDataset, mandatoryQuizOutcomesDataset, From 0497f2aca5a817bf090c9b6e2e8dedb00c85fb2c Mon Sep 17 00:00:00 2001 From: Nicholas Lim Date: Wed, 1 Jul 2026 17:58:51 +0800 Subject: [PATCH 6/8] feat(export): add export orchestrator and entrypoint Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/export-nlds.test.ts | 51 +++++++++++++++++++++++++++++++++++ scripts/export-nlds.ts | 53 ++++++++++++++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/scripts/export-nlds.test.ts b/scripts/export-nlds.test.ts index 13ddbb41..88a8901f 100644 --- a/scripts/export-nlds.test.ts +++ b/scripts/export-nlds.test.ts @@ -6,6 +6,7 @@ import { mandatoryQuizOutcomesDataset, objectKey, putObject, + runExport, toNdjson, usersDataset, } from './export-nlds.js'; @@ -190,3 +191,53 @@ describe('putObject', () => { }); }); }); + +describe('runExport', () => { + const cfg = { + postgresUrl: 'x', + bucket: 'nlds-bucket', + prefix: 'glow/', + kmsKeyId: 'arn:kms:key', + region: 'ap-southeast-1', + }; + + test('uploads both datasets under dated keys and returns their row counts', async () => { + const client = { + user: { + findMany: vi + .fn() + .mockResolvedValue([ + { id: 'u1', name: 'Ada', email: 'a@x', createdAt: new Date('2026-01-01T00:00:00Z') }, + ]), + }, + learningJourney: { findMany: vi.fn().mockResolvedValue([]) }, + } as unknown as import('../src/generated/prisma/client.js').PrismaClient; + const send = vi.fn().mockResolvedValue({}); + const s3 = { send } as unknown as import('@aws-sdk/client-s3').S3Client; + + const results = await runExport({ client, s3, cfg, now: new Date('2026-07-01T00:00:00Z') }); + + expect(results).toEqual([ + { name: 'users', key: 'glow/users/2026-07-01.ndjson', rowCount: 1 }, + { + name: 'mandatory_quiz_outcomes', + key: 'glow/mandatory_quiz_outcomes/2026-07-01.ndjson', + rowCount: 0, + }, + ]); + expect(send).toHaveBeenCalledTimes(2); + }); + + test('propagates an upload failure (aborting the run)', async () => { + const client = { + user: { findMany: vi.fn().mockResolvedValue([]) }, + learningJourney: { findMany: vi.fn().mockResolvedValue([]) }, + } as unknown as import('../src/generated/prisma/client.js').PrismaClient; + const send = vi.fn().mockRejectedValue(new Error('AccessDenied')); + const s3 = { send } as unknown as import('@aws-sdk/client-s3').S3Client; + + await expect( + runExport({ client, s3, cfg, now: new Date('2026-07-01T00:00:00Z') }), + ).rejects.toThrow('AccessDenied'); + }); +}); diff --git a/scripts/export-nlds.ts b/scripts/export-nlds.ts index e8664421..536ac730 100644 --- a/scripts/export-nlds.ts +++ b/scripts/export-nlds.ts @@ -1,6 +1,10 @@ +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { PrismaPg } from '@prisma/adapter-pg'; -import { type PrismaClient } from '../src/generated/prisma/client.js'; +import { PrismaClient } from '../src/generated/prisma/client.js'; export interface ExportConfig { postgresUrl: string; @@ -138,3 +142,50 @@ export const datasets: readonly Dataset>[] = [ usersDataset, mandatoryQuizOutcomesDataset, ]; + +export interface DatasetResult { + name: string; + key: string; + rowCount: number; +} + +export async function runExport(deps: { + client: PrismaClient; + s3: S3Client; + cfg: ExportConfig; + now: Date; +}): Promise { + const { client, s3, cfg, now } = deps; + const date = formatDateUtc(now); + const results: DatasetResult[] = []; + for (const dataset of datasets) { + const rows = await dataset.fetch(client); + const key = objectKey(cfg.prefix, dataset.name, date); + await putObject(s3, cfg, key, toNdjson(rows)); + results.push({ name: dataset.name, key, rowCount: rows.length }); + } + return results; +} + +async function main(): Promise { + const cfg = loadConfig(process.env); + const client = new PrismaClient({ + adapter: new PrismaPg({ connectionString: cfg.postgresUrl }), + }); + const s3 = new S3Client({ region: cfg.region }); + try { + const results = await runExport({ client, s3, cfg, now: new Date() }); + for (const r of results) { + console.log(`exported ${r.rowCount} rows -> s3://${cfg.bucket}/${r.key}`); + } + } finally { + await client.$disconnect(); + } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error('NLDS export failed:', err); + process.exit(1); + }); +} From de7a1c6344f3d9edc0176752620bd6fcf485421c Mon Sep 17 00:00:00 2001 From: Nicholas Lim Date: Wed, 1 Jul 2026 18:22:02 +0800 Subject: [PATCH 7/8] chore(export): wire export:nlds command and build-time bundling Co-Authored-By: Claude Opus 4.8 (1M context) --- Dockerfile | 3 +++ package.json | 3 +++ pnpm-lock.yaml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/Dockerfile b/Dockerfile index 06d5e83f..3301a624 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,9 @@ RUN pnpm db:generate # Build the app. RUN pnpm build +# Bundle the NLDS export script into build/ (see ADR-0006). +RUN pnpm export:nlds:bundle + # Install production dependencies. RUN find . -type d -name "node_modules" -prune -exec rm -rf {} + RUN pnpm install --offline --prod diff --git a/package.json b/package.json index 9de3dd5f..408f13b9 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,8 @@ "check": "svelte-kit sync && svelte-check --fail-on-warnings", "format": "prettier --check \"**/*.{svelte,js,ts,md,html,css,json,yaml}\"", "lint": "eslint \"**/*.{svelte,js,ts}\"", + "export:nlds:bundle": "esbuild scripts/export-nlds.ts --bundle --platform=node --format=esm --target=node24 --packages=external --outfile=build/export-nlds.js", + "export:nlds": "node build/export-nlds.js", "prepare": "svelte-kit sync || echo '' && node \"./.husky/install.js\"" }, "dependencies": { @@ -32,6 +34,7 @@ }, "devDependencies": { "@eslint/js": "^9.39.3", + "esbuild": "^0.28.1", "@lucide/svelte": "^0.511.0", "@sveltejs/adapter-node": "^5.2.12", "@sveltejs/enhanced-img": "^0.8.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9ad8caa2..4c4f2ba1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,6 +48,9 @@ importers: specifier: ^3.13.0 version: 3.13.0 devDependencies: + esbuild: + specifier: ^0.28.1 + version: 0.28.1 '@eslint/js': specifier: ^9.39.3 version: 9.39.3 From 5a5524aa9a5c61313fcfddf42cacefab548f6b0d Mon Sep 17 00:00:00 2001 From: Nicholas Lim Date: Wed, 1 Jul 2026 18:28:17 +0800 Subject: [PATCH 8/8] refactor(export): reuse formatDateUtc for due_date and test prefix scoping Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/export-nlds.test.ts | 8 ++++++++ scripts/export-nlds.ts | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/export-nlds.test.ts b/scripts/export-nlds.test.ts index 88a8901f..508b9e47 100644 --- a/scripts/export-nlds.test.ts +++ b/scripts/export-nlds.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test, vi } from 'vitest'; import { + datasets, formatDateUtc, loadConfig, mandatoryQuizOutcomesDataset, @@ -45,6 +46,13 @@ describe('objectKey', () => { test('composes prefix + dataset + date + .ndjson', () => { expect(objectKey('glow/', 'users', '2026-07-01')).toBe('glow/users/2026-07-01.ndjson'); }); + + test('composes keys that stay within the prefix for every dataset', () => { + for (const dataset of datasets) { + const key = objectKey('glow/', dataset.name, '2026-07-01'); + expect(key.startsWith('glow/')).toBe(true); + } + }); }); describe('formatDateUtc', () => { diff --git a/scripts/export-nlds.ts b/scripts/export-nlds.ts index 536ac730..b0d831d9 100644 --- a/scripts/export-nlds.ts +++ b/scripts/export-nlds.ts @@ -110,7 +110,7 @@ export const mandatoryQuizOutcomesDataset: Dataset = { user_id: r.userId, learning_unit_id: r.learningUnitId, learning_unit_title: r.learningUnit.title, - due_date: r.learningUnit.dueDate ? r.learningUnit.dueDate.toISOString().slice(0, 10) : null, + due_date: r.learningUnit.dueDate ? formatDateUtc(r.learningUnit.dueDate) : null, is_completed: r.isCompleted, is_quiz_attempted: r.isQuizAttempted, is_quiz_passed: r.isQuizPassed,