From 381f58c5373b446db0fdf76ed2270eafc01a2088 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 23 Jun 2026 04:39:36 +0000 Subject: [PATCH 1/6] Initial plan From 6ffc46a90a8b7f9a47ee2fc38977fcaf66877a3b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 23 Jun 2026 05:04:23 +0000 Subject: [PATCH 2/6] chore: initial plan for monolithic refactoring --- package-lock.json | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index aed453110..cd4bc9767 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2094,9 +2094,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2114,9 +2111,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2134,9 +2128,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2154,9 +2145,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2174,9 +2162,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2194,9 +2179,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ From 4786cd2de8e78c62c02bca474b0c917151620281 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 23 Jun 2026 05:06:48 +0000 Subject: [PATCH 3/6] Changes before error encountered Agent-Logs-Url: https://github.com/BigSimmo/Database/sessions/8163c95b-f171-4a6d-9474-cd111763c4a8 --- worker/utils.ts | 128 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 worker/utils.ts diff --git a/worker/utils.ts b/worker/utils.ts new file mode 100644 index 000000000..d3233c0f5 --- /dev/null +++ b/worker/utils.ts @@ -0,0 +1,128 @@ +/** + * Pure utility functions shared across worker stage modules. + * + * These helpers have no side-effects and do not touch Supabase or OpenAI. + * They can be imported freely by any worker module without risk of circular + * dependencies or unexpected I/O. + */ + +import { createHash } from "node:crypto"; +import { env } from "../src/lib/env"; +import { safeErrorLogDetails } from "../src/lib/privacy"; + +// --------------------------------------------------------------------------- +// String helpers +// --------------------------------------------------------------------------- + +/** Strip null bytes and normalise surrogate pairs so Postgres accepts the value. */ +export function cleanString(val: string): string { + if (typeof val !== "string") return val; + return val + .replace(/\u0000/g, "") + .replace(/\\u0000/g, "") + .toWellFormed(); +} + +// --------------------------------------------------------------------------- +// JSONB helpers +// --------------------------------------------------------------------------- + +export type JsonbValue = string | number | boolean | null | { [key: string]: JsonbValue } | JsonbValue[]; +export type JsonbRecord = { [key: string]: JsonbValue }; + +/** Recursively clean a value so it is safe to store as JSONB in Postgres. */ +export function sanitizeJsonb(val: unknown): JsonbValue { + if (typeof val === "string") return cleanString(val); + if (Array.isArray(val)) return val.map((entry) => sanitizeJsonb(entry)); + if (val !== null && typeof val === "object") { + const raw = val as { [key: string]: unknown }; + const res: JsonbRecord = {}; + for (const [key, value] of Object.entries(raw)) { + res[key] = sanitizeJsonb(value); + } + return res; + } + return val as JsonbValue; +} + +/** Return a sanitised JSONB-safe record, falling back to `{}` for non-objects. */ +export function sanitizeJsonbRecord(value: unknown): Record { + const sanitized = sanitizeJsonb(value); + return typeof sanitized === "object" && sanitized !== null && !Array.isArray(sanitized) ? sanitized : {}; +} + +// --------------------------------------------------------------------------- +// Hash helpers +// --------------------------------------------------------------------------- + +export function hashBytes(bytes: Buffer) { + return createHash("sha256").update(bytes).digest("hex"); +} + +export function hashText(text: string) { + return createHash("sha256").update(text.replace(/\s+/g, " ").trim()).digest("hex"); +} + +export function hashEmbeddingFieldContent(content: string) { + return createHash("md5").update(content).digest("hex"); +} + +// --------------------------------------------------------------------------- +// Text helpers +// --------------------------------------------------------------------------- + +/** + * Collapse whitespace and truncate to `limit` characters, preserving whole + * words where possible. + */ +export function compactSearchText(value: unknown, limit = 900) { + const compact = String(value ?? "") + .replace(/\s+/g, " ") + .trim(); + if (!compact) return ""; + return compact.length > limit ? compact.slice(0, limit).trim() : compact; +} + +// --------------------------------------------------------------------------- +// Worker-specific helpers +// --------------------------------------------------------------------------- + +/** Increment the count for a skip reason in the shared skip-reason map. */ +export function noteSkippedImage(skipReasons: Map, reason: string) { + skipReasons.set(reason, (skipReasons.get(reason) ?? 0) + 1); +} + +/** + * Return `true` when the Supabase error indicates a missing RPC / schema + * mismatch so the worker can decide whether to retry after a migration. + */ +export function isMissingSchemaError(error: { message?: string; code?: string }) { + return /could not find the function|schema cache|PGRST20\d/i.test(error.message ?? "") || error.code === "PGRST202"; +} + +/** Calculate an exponential back-off delay for the polling loop. */ +export function workerBackoffMs(failures: number) { + return Math.min(env.WORKER_HEALTH_BACKOFF_MS, env.WORKER_POLL_MS * 2 ** Math.max(0, failures - 1)); +} + +/** Log a non-fatal write failure without propagating the error. */ +export function optionalIndexWriteWarning(stage: string, error: unknown) { + console.warn(`Optional ${stage} write failed`, safeErrorLogDetails(error)); +} + +/** + * Create a structured Error from a Supabase response error, preserving the + * Postgres error code, details and hint for upstream logging. + */ +export function supabaseStageError( + stage: string, + error: { message?: string; code?: string; details?: string; hint?: string }, +) { + const wrapped = new Error(`${stage}: ${error.message ?? "Supabase request failed"}`); + Object.assign(wrapped, { + code: error.code, + details: error.details, + hint: error.hint, + }); + return wrapped; +} From 2b0c01da11df183533ae4c84ffa705311bf4e46c Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:50:14 +0800 Subject: [PATCH 4/6] Delete .github/workflows/label.yml --- .github/workflows/label.yml | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .github/workflows/label.yml diff --git a/.github/workflows/label.yml b/.github/workflows/label.yml deleted file mode 100644 index 461356907..000000000 --- a/.github/workflows/label.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This workflow will triage pull requests and apply a label based on the -# paths that are modified in the pull request. -# -# To use this workflow, you will need to set up a .github/labeler.yml -# file with configuration. For more information, see: -# https://github.com/actions/labeler - -name: Labeler -on: [pull_request_target] - -jobs: - label: - - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - - steps: - - uses: actions/labeler@v4 - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" From 80e36a10031fb188f0e9d9870e3064e8e8d75bf2 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:50:25 +0800 Subject: [PATCH 5/6] Delete .github/workflows/node.js.yml --- .github/workflows/node.js.yml | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 .github/workflows/node.js.yml diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml deleted file mode 100644 index 2284b9357..000000000 --- a/.github/workflows/node.js.yml +++ /dev/null @@ -1,31 +0,0 @@ -# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs - -name: Node.js CI - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - build: - - runs-on: ubuntu-latest - - strategy: - matrix: - node-version: [18.x, 20.x, 22.x] - # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ - - steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - cache: 'npm' - - run: npm ci - - run: npm run build --if-present - - run: npm test From 76e9b609ed6004815ef89772b78f3eab4f7106c2 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:50:41 +0800 Subject: [PATCH 6/6] Delete .github/workflows/codeql.yml --- .github/workflows/codeql.yml | 104 ----------------------------------- 1 file changed, 104 deletions(-) delete mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index bb7fb9baf..000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,104 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL Advanced" - -on: - push: - branches: ["main", "protection"] - pull_request: - branches: ["main", "protection"] - schedule: - - cron: "20 12 * * 2" - workflow_dispatch: - -jobs: - analyze: - name: Analyze (${{ matrix.language }}) - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners (GitHub.com only) - # Consider using larger runners or machines with greater resources for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - permissions: - # required for all workflows - security-events: write - - # required to fetch internal or private CodeQL packs - packages: read - - # only required for workflows in private repositories - actions: read - contents: read - - strategy: - fail-fast: false - matrix: - include: - - language: actions - build-mode: none - - language: javascript-typescript - build-mode: none - - language: python - build-mode: none - # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift' - # Use `c-cpp` to analyze code written in C, C++ or both - # Use 'java-kotlin' to analyze code written in Java, Kotlin or both - # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both - # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, - # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. - # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how - # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - # Add any setup steps before running the `github/codeql-action/init` action. - # This includes steps like installing compilers or runtimes (`actions/setup-node` - # or others). This is typically only required for manual builds. - # - name: Setup runtime (example) - # uses: actions/setup-example@v1 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: ${{ matrix.language }} - build-mode: ${{ matrix.build-mode }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - # If the analyze step fails for one of the languages you are analyzing with - # "We were unable to automatically build your code", modify the matrix above - # to set the build mode to "manual" for that language. Then modify this step - # to build your code. - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - name: Run manual build steps - if: matrix.build-mode == 'manual' - shell: bash - run: | - echo 'If you are using a "manual" build mode for one or more of the' \ - 'languages you are analyzing, replace this with the commands to build' \ - 'your code, for example:' - echo ' make bootstrap' - echo ' make release' - exit 1 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 - with: - category: "/language:${{matrix.language}}"