From 77d95b248589fc5c538391bcb5cd8948f1eb4c45 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Sun, 19 Jul 2026 03:19:57 -0400 Subject: [PATCH 1/2] fix: harden legacy storage migration --- .changeset/clean-cups-help.md | 6 + README.md | 8 + apps/docs/astro.config.mjs | 20 +- .../src/content/docs/agent-integrations.mdx | 20 ++ apps/docs/src/content/docs/catalog.mdx | 13 +- apps/docs/src/content/docs/configuration.mdx | 7 + apps/docs/src/content/docs/dashboard.mdx | 10 +- apps/docs/src/content/docs/index.mdx | 10 + apps/docs/src/content/docs/install.mdx | 40 ++- apps/docs/src/content/docs/privacy.mdx | 111 ++++++++ .../src/content/docs/privacy/indexing.mdx | 38 +-- .../docs/src/content/docs/project-binding.mdx | 120 +++++++++ apps/docs/src/content/docs/reference/cli.mdx | 214 ++++++++++++++++ .../src/content/docs/reference/config.mdx | 32 +++ apps/docs/src/content/docs/remote-attach.mdx | 6 +- apps/docs/src/content/docs/storage.mdx | 186 ++++++++++++++ .../docs/src/content/docs/troubleshooting.mdx | 52 +++- apps/docs/src/content/docs/vault.mdx | 24 +- .../sql-authoritative-host-state.md | 37 ++- packages/cli/CHANGELOG.md | 6 + packages/core/CHANGELOG.md | 6 + packages/core/src/cli.ts | 38 ++- packages/core/src/storage/legacy-migration.ts | 105 ++++---- packages/core/src/storage/vault-grants.ts | 242 ++++++++++-------- packages/core/test/legacy-migration.test.ts | 30 +++ .../core/test/storage-records-cli.test.ts | 130 ++++++++-- scripts/check-public-docs.ts | 26 ++ scripts/generate-docs-reference.ts | 38 +++ 28 files changed, 1324 insertions(+), 251 deletions(-) create mode 100644 .changeset/clean-cups-help.md create mode 100644 apps/docs/src/content/docs/privacy.mdx create mode 100644 apps/docs/src/content/docs/project-binding.mdx create mode 100644 apps/docs/src/content/docs/reference/cli.mdx create mode 100644 apps/docs/src/content/docs/storage.mdx diff --git a/.changeset/clean-cups-help.md b/.changeset/clean-cups-help.md new file mode 100644 index 00000000..9cd5d8fe --- /dev/null +++ b/.changeset/clean-cups-help.md @@ -0,0 +1,6 @@ +--- +"@caplets/core": patch +"caplets": patch +--- + +**Required upgrade migration:** Hosts that ran `caplets@0.25.x` or earlier must stop every Caplets Host Node and run `caplets storage migrate-legacy --dry-run`, then `caplets storage migrate-legacy`, before restarting the daemon, running `caplets setup`, or serving requests. The migration now imports standard legacy auth, Vault, remote security, setup, Operator Activity, and tracked-Caplet state; preserves file-layer Vault grants and shared Vault key access; uses platform paths by default; and reports actionable missing tracked entries. diff --git a/README.md b/README.md index fc2ee61f..4245334c 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,14 @@ Caplets can wrap: Full setup and configuration docs are available at [docs.caplets.dev](https://docs.caplets.dev/). +> **Required upgrade migration:** If this host ran `caplets@0.25.x` or earlier, installing +> `caplets@0.26.0` or later requires a one-time SQL migration **before** you restart the +> daemon, run `caplets setup`, or serve requests. Stop every Caplets Host Node, then run +> `caplets storage migrate-legacy --dry-run` followed by +> `caplets storage migrate-legacy`. Skipping this step leaves legacy auth, Vault, remote +> security, setup, activity, and tracked-Caplet state outside the authoritative SQL store. +> See [SQL Authoritative Host State operations](docs/operations/sql-authoritative-host-state.md#offline-legacy-filesystem-migration). + Install the CLI and wire it into your agent: ```sh diff --git a/apps/docs/astro.config.mjs b/apps/docs/astro.config.mjs index f3404daa..1062f019 100644 --- a/apps/docs/astro.config.mjs +++ b/apps/docs/astro.config.mjs @@ -41,7 +41,6 @@ export default defineConfig({ items: [ { label: "Quick Start", link: "/" }, { label: "Install", link: "/install/" }, - { label: "Configuration", link: "/configuration/" }, ], }, { @@ -49,20 +48,35 @@ export default defineConfig({ items: [ { label: "Code Mode", link: "/code-mode/" }, { label: "Add capabilities", link: "/capabilities/" }, + { label: "Configuration", link: "/configuration/" }, { label: "Catalog", link: "/catalog/" }, - { label: "Dashboard", link: "/dashboard/" }, { label: "Caplets Vault", link: "/vault/" }, + { label: "Project Binding", link: "/project-binding/" }, + ], + }, + { + label: "Connect Agents", + items: [ { label: "Agent integrations", link: "/agent-integrations/" }, { label: "Remote attach", link: "/remote-attach/" }, + ], + }, + { + label: "Operate a Host", + items: [ + { label: "Dashboard", link: "/dashboard/" }, + { label: "Authoritative Host State", link: "/storage/" }, + { label: "Privacy and network activity", link: "/privacy/" }, { label: "Troubleshooting", link: "/troubleshooting/" }, ], }, { label: "Reference", items: [ + { label: "CLI", link: "/reference/cli/" }, { label: "Configuration schema", link: "/reference/config/" }, - { label: "Code Mode API", link: "/reference/code-mode-api/" }, { label: "Caplet files", link: "/reference/caplet-files/" }, + { label: "Code Mode API", link: "/reference/code-mode-api/" }, { label: "Catalog indexing privacy", link: "/privacy/indexing/" }, { label: "GitHub releases", diff --git a/apps/docs/src/content/docs/agent-integrations.mdx b/apps/docs/src/content/docs/agent-integrations.mdx index 03c15219..7347becd 100644 --- a/apps/docs/src/content/docs/agent-integrations.mdx +++ b/apps/docs/src/content/docs/agent-integrations.mdx @@ -79,6 +79,22 @@ settings still win over those defaults. Native integrations use the shared anonymous telemetry controls and do not send telemetry on a native-first run until a visible CLI telemetry notice has already been recorded. +### Native runtime selection + +OpenCode and Pi accept shared runtime selectors through integration settings or environment: + +| Mode | Selection | +| -------- | ---------------------------------------------------------------------------------------------------- | +| `auto` | Cloud for Cloud URLs, remote for other URLs, setup-written daemon defaults when present, then local. | +| `local` | Run Caplets in the agent process with user and project config. | +| `daemon` | Use the loopback runtime selected by `CAPLETS_DAEMON_URL`. | +| `remote` | Use the saved Remote Profile selected by `CAPLETS_REMOTE_URL`. | +| `cloud` | Use the Cloud runtime selected by `CAPLETS_REMOTE_URL` and optional `CAPLETS_REMOTE_WORKSPACE`. | + +Cloud native sessions and loopback daemon sessions with a project root can open +[Project Binding](/project-binding/) for required Caplets. Self-hosted HTTPS native remotes do not +currently establish that managed binding session; required project Caplets remain hidden. + Remote native usage: ```sh @@ -122,3 +138,7 @@ caplets serve --transport http --upstream-url https://caplets.example.com/caplet Then configure the MCP client with `caplets attach `. If you also need host administration, open the same runtime's `/dashboard` path in a browser and complete an Operator approval there. + +Use the [CLI reference](/reference/cli/) for the complete command-family map and +[Privacy and network activity](/privacy/) for the independent native telemetry, update-check, +and catalog-indexing controls. diff --git a/apps/docs/src/content/docs/catalog.mdx b/apps/docs/src/content/docs/catalog.mdx index 35578959..2e4b87cd 100644 --- a/apps/docs/src/content/docs/catalog.mdx +++ b/apps/docs/src/content/docs/catalog.mdx @@ -6,8 +6,8 @@ description: Search public Caplets and inspect installable catalog entries. catalog.caplets.dev -is the public search surface for Caplets. It lists official Caplets from this repository and public -community Caplets discovered from successful public external installs. +is the public search surface for Caplets. It lists official Caplets from this repository and +community Caplets submitted after eligible GitHub-shaped external installs. --yes ``` -If the host uses a non-default remote credential state directory, use the matching `--state-path` -value when approving from the CLI. +The deprecated `--state-path` option no longer selects credential state. Remote clients, +pending logins, and dashboard sessions are stored in the host's configured SQL backend. After approval, the browser receives an operator dashboard session and can administer the Current Host. diff --git a/apps/docs/src/content/docs/index.mdx b/apps/docs/src/content/docs/index.mdx index 5e353c89..190a7751 100644 --- a/apps/docs/src/content/docs/index.mdx +++ b/apps/docs/src/content/docs/index.mdx @@ -10,6 +10,12 @@ handle that an agent can inspect, search, call, filter, and summarize in one wor Use Caplets with Codex, Claude, OpenCode, Pi, or any MCP client that can launch a local stdio server. +> **Required upgrade migration:** If this host ran `caplets@0.25.x` or earlier, do not +> start `caplets@0.26.0` or later until you complete the +> [one-time legacy-state migration](/install/#required-migration-when-upgrading-to-0260-or-later). +> Caplets does not automatically migrate or fall back to the old auth, Vault, remote-security, +> setup, activity, and tracked-Caplet stores. + ## Quick Start Install the CLI, then run a known-good no-auth setup first. OSV is public, so it is the @@ -68,4 +74,8 @@ should contain the OSV identifiers the agent found. - [Caplets Vault](/vault/) - store secret config values without relying on agent environment propagation. - [Agent integrations](/agent-integrations/) - use Codex, Claude, OpenCode, and Pi. - [Remote attach](/remote-attach/) - connect agents to a remote Caplets runtime. +- [Project Binding](/project-binding/) - synchronize local repository context for remote Caplets. +- [Authoritative Host State](/storage/) - operate SQLite or PostgreSQL host state. +- [CLI reference](/reference/cli/) - find command families and environment controls. +- [Privacy and network activity](/privacy/) - control telemetry, update checks, and catalog indexing. - [Troubleshooting](/troubleshooting/) - check config, auth, and runtime issues. diff --git a/apps/docs/src/content/docs/install.mdx b/apps/docs/src/content/docs/install.mdx index d2c32033..4fc70be4 100644 --- a/apps/docs/src/content/docs/install.mdx +++ b/apps/docs/src/content/docs/install.mdx @@ -5,9 +5,36 @@ description: Install Caplets and wire it into your agent. Caplets is pre-1.0. Use the latest CLI and latest docs together. +## Required migration when upgrading to 0.26.0 or later + +> **Do not start the upgraded host first.** If this host ran `caplets@0.25.x` or earlier, +> the one-time legacy-state migration is required after installing `caplets@0.26.0` or +> later and before restarting the daemon, running `caplets setup`, or serving requests. + +Caplets 0.26.0 moved Authoritative Host State into SQL. It does not automatically migrate +or fall back to the old backend auth, Vault, remote-security, setup, Operator Activity, or +tracked-Caplet stores. Skipping migration can therefore make existing credentials, Vault +values and grants, remote clients, setup history, activity, and tracked Caplets appear +missing. + +Stop every Caplets Host Node, back up the legacy state roots and configured SQL database, +then verify and run the migration: + +```sh +caplets storage migrate-legacy --dry-run +caplets storage migrate-legacy +caplets storage status --json +``` + +Resolve every dry-run error before running the write migration. Record the backup path +printed by the migration, and restart the host only after storage status reports ready. +Fresh hosts that never ran a version before 0.26.0 do not need this migration. See +[Authoritative Host State](/storage/#upgrade-from-025x-or-earlier) for default paths, +custom-path guidance, backup requirements, and recovery behavior. + ## Requirements -- Node.js 24 or newer for published packages. +- Node.js 22 or newer for published packages. - An agent that can use MCP, such as Codex or Claude, or a native integration such as OpenCode or Pi. @@ -72,11 +99,12 @@ caplets update sentry github Install, restore, and update refuse to overwrite local edits unless you pass `--force`. Use `--json` when a script needs per-entry statuses and machine-readable errors. -For public external sources, install, restore, and update also attempt best-effort catalog -indexing after the local mutation succeeds. Human output prints a short notice when a -public Caplet source may be indexed. JSON output includes per-entry `catalogIndexing` -status values. Indexing failures never block install/update/restore, and private or local -sources are skipped with redacted categorical statuses. +For eligible GitHub-shaped external sources, install, restore, and update also attempt +best-effort catalog indexing after the local mutation succeeds. Human output prints a short +notice when a source may be indexed. JSON output includes per-entry `catalogIndexing` status +values. Eligibility does not verify GitHub repository visibility, and the payload includes +the installed `CAPLET.md`; disable indexing for private repositories or non-public content. +Indexing failures never block install/update/restore. Set `CAPLETS_DISABLE_CATALOG_INDEXING=1` to skip catalog indexing network requests entirely. diff --git a/apps/docs/src/content/docs/privacy.mdx b/apps/docs/src/content/docs/privacy.mdx new file mode 100644 index 00000000..0e80e721 --- /dev/null +++ b/apps/docs/src/content/docs/privacy.mdx @@ -0,0 +1,111 @@ +--- +title: Privacy and Network Activity +description: Understand and control telemetry, update checks, and catalog indexing. +--- + +Caplets has three independent outbound product services: anonymous telemetry, passive npm update +metadata checks, and best-effort public catalog indexing after eligible lifecycle operations. Each +has a separate control. Disabling one does not disable the others. + +Remote Caplets, downstream APIs, OAuth providers, and user-configured catalog sources naturally add +the network calls required by those capabilities; those are not product telemetry. + +## Anonymous telemetry + +Telemetry is opt-out and reports bounded product-usage and reliability categories. The first +eligible interactive CLI run prints a notice to stderr. Non-CI telemetry remains suppressed until +that notice has been recorded, so a native-first run does not silently activate collection. + +Disable telemetry for one process: + +```sh +CAPLETS_DISABLE_TELEMETRY=1 caplets serve +``` + +Disable it in user config and inspect the effective state: + +```sh +caplets telemetry disable +caplets telemetry status +``` + +Re-enable it with `caplets telemetry enable`. The environment variable wins over user config. +Tests disable telemetry automatically. + +Telemetry properties are allowlisted categorical values, integer counts, booleans, duration/timeout +buckets, package/version metadata, and coarse runtime information such as OS family, architecture, +and Node major version. Reliability exceptions keep only sanitized package/workspace-relative stack +frames and an error type. + +Caplets telemetry does **not** collect raw config, prompts, Code Mode source, tool arguments, tool +outputs, resource or prompt contents, logs, raw or absolute local paths, URLs, hostnames, Caplet IDs, +credentials, tokens, raw environment variables, raw error messages, or unsanitized stack traces. +Reliability stack frames may retain the sanitized package/workspace-relative paths or safe basenames +described above. + +Inspect the sanitized events a command would produce without delivering them: + +```sh +caplets telemetry debug -- doctor +caplets telemetry debug -- code-mode 'return 1' +``` + +Telemetry debug mode also disables the update check for that invocation. Manage the anonymous local +identity separately: + +```sh +caplets telemetry rotate-id +caplets telemetry delete-id +``` + +## Passive update checks + +Eligible CLI commands may fetch public package metadata from +`https://registry.npmjs.org/caplets`. The lookup requests only npm's install metadata, has a 250 ms +fetch timeout and 1 MiB response limit, and keeps successful metadata fresh for 24 hours. If later +refreshes fail, that metadata may remain usable for a further seven days. An update notice is +written to stderr at most once per version every seven days. + +Update notices avoid help, version, completion, JSON/output-product commands, Code Mode, daemon +commands, CI, and non-interactive output. Stdio `serve` and `attach` stay quiet by default so MCP +protocol output is not polluted. Set `CAPLETS_UPDATE_NOTICE_STDERR=1` only for a foreground stdio +process whose stderr is visible and separate from protocol stdout. + +Disable both update notices and metadata lookups: + +```sh +CAPLETS_DISABLE_UPDATE_CHECK=1 caplets doctor +``` + +This switch is independent from telemetry. + +## Public catalog indexing + +After a successful install, restore, or update of an eligible GitHub-shaped external Caplet, +Caplets may submit normalized source identity, revisions and hashes, the full installed +`CAPLET.md`, and derived catalog/child metadata to the public catalog. Eligibility checks source +syntax but do not verify GitHub repository visibility. Disable indexing before operating on a +private GitHub repository or any Caplet file whose contents must not become public. Indexing is +best effort and never changes the local lifecycle result. + +Disable catalog indexing network requests: + +```sh +CAPLETS_DISABLE_CATALOG_INDEXING=1 caplets install spiritledsoftware/caplets osv +``` + +The envelope does not separately add installer identity, local filesystem paths, Vault values, +agent prompts, tool arguments, tool outputs, or raw environment values. Because it includes the +full Caplet file, anything authored in that file can be submitted. See +[Catalog indexing privacy](/privacy/indexing/) for the exact boundary. + +## Control matrix + +| Activity | Default | Disable | +| ----------------------- | ----------------------------------------------------- | ------------------------------------------------------------ | +| Anonymous telemetry | Opt-out after the visible notice gate | `CAPLETS_DISABLE_TELEMETRY=1` or `caplets telemetry disable` | +| Passive update metadata | Eligible foreground CLI commands | `CAPLETS_DISABLE_UPDATE_CHECK=1` | +| Public catalog indexing | Eligible GitHub-shaped install/restore/update results | `CAPLETS_DISABLE_CATALOG_INDEXING=1` | + +For machine-readable automation, prefer `--json` where supported. Caplets keeps update notices and +telemetry notices on stderr and suppresses them for structured output paths. diff --git a/apps/docs/src/content/docs/privacy/indexing.mdx b/apps/docs/src/content/docs/privacy/indexing.mdx index 58d59277..f6678578 100644 --- a/apps/docs/src/content/docs/privacy/indexing.mdx +++ b/apps/docs/src/content/docs/privacy/indexing.mdx @@ -9,8 +9,8 @@ Set `CAPLETS_DISABLE_CATALOG_INDEXING=1` to disable catalog indexing submissions Anonymous telemetry intentionally excludes source URLs, Caplet IDs, hostnames, local paths, tool arguments, tool outputs, raw config, raw search text, session replay, and raw browser URLs. -Public catalog indexing needs different data: it publishes public-source metadata so other users -can search for public Caplets. +Public catalog indexing submits source and Caplet content so other users can search public +Caplets. It is a separate disclosure boundary from anonymous telemetry. Landing, docs, and catalog analytics are anonymous telemetry, not public catalog indexing. They use categorical page, search, filter, and install-intent events. A final browser PostHog envelope @@ -21,25 +21,31 @@ known-user or person-profile attribution, and they never hand browser identity i ## What can become public -After a successful install, restore, or update of an eligible public external Caplet, +After a successful install, restore, or update of an eligible GitHub-shaped external Caplet, Caplets may submit: -- Normalized public provider and repository identity, such as `github.com/owner/repo`. -- Caplet ID and source path inside the public repository. -- Resolved revision or content hash when available. -- Installed content hash. -- Generated install command metadata. -- Safe derived catalog metadata, warnings, and aggregate install counts. +- Normalized provider and repository identity, such as `github.com/owner/repo`. +- Caplet ID and source path inside the repository. +- Resolved revision and installed content hash. +- The full installed `CAPLET.md` content. +- Derived name, description, icon, tags, setup/auth/Project Binding requirements, workflow and + risk summaries, child metadata, warnings, and generated install command data. -## What must not be sent +Eligibility validates GitHub source syntax, not repository visibility. A private GitHub repository +can therefore be eligible. Disable catalog indexing before installing from a private repository or +when any Caplet file contains material that must not become public. -Catalog indexing must not send installer identity, local paths, private config, credentials, -raw agent prompts, tool arguments, tool outputs, hostnames for private sources, private -source URLs, Vault secret values, raw environment values, or individual install-event -identity. +## Envelope boundary -Ineligible sources return categorical statuses such as `ineligible`, -`revision_unavailable`, or `unavailable` without echoing raw private source values. +The indexing envelope does not separately add installer identity, local filesystem paths, Vault +values, raw environment values, agent prompts, tool arguments, tool outputs, or individual +install-event identity. It rejects credential-bearing source URLs and private-network source +hosts. + +The full Caplet file is still part of the envelope. Anything authored in its Markdown or +frontmatter—including literal secrets, internal URLs, or private hostnames—can be submitted. Do +not put secrets in Caplet files, and use `CAPLETS_DISABLE_CATALOG_INDEXING=1` whenever the source +or content is not intended for publication. ## Nonblocking behavior diff --git a/apps/docs/src/content/docs/project-binding.mdx b/apps/docs/src/content/docs/project-binding.mdx new file mode 100644 index 00000000..381f2f57 --- /dev/null +++ b/apps/docs/src/content/docs/project-binding.mdx @@ -0,0 +1,120 @@ +--- +title: Project Binding +description: Bind a local project to remote Caplets that need repository context. +--- + +Project Binding lets a supported Access Client keep a local repository synchronized with a remote +Caplets runtime. Required Caplets stay hidden unless the session supplies a ready binding context; +Caplets never silently runs them in the server startup directory. + +A Caplet opts into this requirement explicitly: + +```json +{ + "cliTools": { + "repo-checks": { + "name": "Repository checks", + "description": "Run approved checks in the attached project.", + "projectBinding": { "required": true }, + "actions": { + "test": { "command": "pnpm", "args": ["test"] } + } + } + } +} +``` + +The released managed session path is wired for native Caplets Cloud sessions and native loopback +daemon sessions that provide a project root. A self-hosted HTTPS native remote does not currently +establish a managed Project Binding session. + +## How a binding starts + +Native OpenCode and Pi integrations can pass project context in Cloud and loopback daemon modes. +The binding session establishes or renews a lease and waits for managed project sync before +required Caplets are exposed. + +The ordinary `caplets attach ` command is a thin stdio MCP adapter; it does not itself run +managed project synchronization. Its `--once` mode is the CLI preflight for a remote binding +endpoint: + +```sh +cd /path/to/project +caplets remote login https://caplets.example.com/caplets +caplets attach https://caplets.example.com/caplets --once +``` + +A stacked runtime also attempts upstream Project Binding when a client supplies project context and +the upstream supports it. If the upstream binding path is unavailable, local project Caplets and +non-project upstream Caplets remain available; required upstream project Caplets stay unavailable +with a diagnostic. + +## Preflight the binding path + +Use the finite smoke path with an explicit remote selector before editing agent configuration: + +```sh +caplets attach https://caplets.example.com/caplets --once +``` + +This builds the local sync manifest, enforces preflight size policy, probes the selected Project +Binding WebSocket endpoint, and exits. It does not start a sync session and is not a generic MCP +health check. + +`caplets doctor` and `caplets doctor --json` report the resolved project root, fingerprint, +workspace path, remote auth mode, WebSocket URL, and recovery command. The standalone CLI does not +receive a live sync snapshot, so its sync fields remain `idle` with default Mutagen diagnostics. + +## What is synchronized + +Caplets builds a manifest before synchronization. It applies: + +1. a hard denylist for VCS internals, dependency/build caches, common secret files, private keys, + archives, and local environment files; +2. supported rules from the project's `.gitignore` and `.git/info/exclude`; and +3. supported rules from optional `.capletsignore`. + +The current matcher supports exact file or directory names, trailing-slash directory rules, +leading `*.` suffix rules, leading-slash root anchoring, and `!` negation. Other glob syntax such +as `**`, embedded `*`, `?`, and character classes is treated literally; do not rely on it to +protect a sync. + +Safe environment templates named `.env.example`, `.env.sample`, or `.env.template` bypass the hard +denylist and `.capletsignore`, but an explicit `.gitignore` rule still excludes them. Keep real +credentials out of templates, and use `.gitignore` or rename the file when one must not sync. +Symbolic links are always skipped. + +Caplets creates `.caplets/.gitignore` with the contents needed to keep binding-local state out of +the repository. It does not use that directory as a source of runtime secrets. + +The CLI sync preflight enforces both total-project and single-file limits. It uses the `free` policy +by default: 25 MiB per file and 250 MiB for the selected project after exclusions. Hosted contexts +may supply `plus` (100 MiB/1 GiB), `pro` (250 MiB/5 GiB), or `enterprise` (250 MiB/5 GiB) policy. +A limit failure returns `sync_size_limit_exceeded`; exclude unnecessary content or change the +applicable workspace plan before retrying. + +## Runtime safety + +- Project-bound working directories are resolved under the synchronized root. A configured `cwd` + that escapes through `..` or a symbolic link is rejected. +- Binding leases and readiness metadata are authoritative host state. Workspace bytes remain on the + owning Host Node. +- If an owning node is lost, required Caplets are quarantined until the client rebinds to a healthy + node and fully resynchronizes. PostgreSQL does not reconstruct workspace bytes. +- A missing binding context hides only Caplets that declare `projectBinding.required`; unrelated + local and upstream Caplets remain usable. + +## Common failures + +| Diagnostic | Meaning | Repair | +| ----------------------------------- | ------------------------------------------------------------------- | ------------------------------------------------------ | +| `remote_credentials_required` | The selected self-hosted remote has no valid Remote Profile. | Run `caplets remote login `. | +| `cloud_auth_required` | Cloud credentials or required scopes are missing. | Log in again to the Cloud URL. | +| `workspace_switch_required` | The requested workspace differs from the saved selection. | Select the intended workspace, then reconnect. | +| `websocket_upgrade_required` | A proxy did not pass the WebSocket upgrade. | Enable WebSocket forwarding for the Caplets base path. | +| `sync_size_limit_exceeded` | The post-exclusion project is too large. | Add a supported ignore rule or change the plan limit. | +| `lease_conflict` or `lease_expired` | Another owner holds the binding or the client stopped heartbeating. | End the stale session or reconnect and resynchronize. | + +Start with `caplets doctor --json`; its Project Binding section carries resolved selection and +recovery fields without exposing credentials. Use the active session's own diagnostics for live +lease and managed-sync state. diff --git a/apps/docs/src/content/docs/reference/cli.mdx b/apps/docs/src/content/docs/reference/cli.mdx new file mode 100644 index 00000000..e11f4cf8 --- /dev/null +++ b/apps/docs/src/content/docs/reference/cli.mdx @@ -0,0 +1,214 @@ +--- +title: CLI Reference +description: Find Caplets command families, subcommands, output modes, and environment selectors. +--- + +The running CLI is the final authority for options in your installed release: + +```sh +caplets --help +caplets --help +caplets --help +``` + +This page maps the command surface by job. It intentionally does not duplicate every flag from +Commander help. + +## Setup and lifecycle + +| Command | Purpose | +| ------------------------------------- | -------------------------------------------------------------------------------------- | +| `caplets init` | Create a starter user, project, or remote config. | +| `caplets setup [integration]` | Configure Codex, Claude Code, OpenCode, Pi, or a cataloged MCP client. | +| `caplets doctor` | Diagnose local config, remote selection, daemon state, auth, and Project Binding. | +| `caplets list` | List configured Caplets; use `--all` to include disabled entries. | +| `caplets install [repo] [caplets...]` | Install from a repository or restore the selected lockfile when the source is omitted. | +| `caplets update [caplets...]` | Update all or selected tracked Caplets. | +| `caplets inspect ` | Print one configured Caplet card. | +| `caplets check-backend ` | Check backend availability and auth readiness. | + +`install` and `update` select project, user/global, or remote lifecycle targets through their flags. +Run help before using `--force`; normal operation refuses to overwrite divergent local content. + +## Add capability files + +`caplets add` writes Markdown Caplet files for local configuration: + +```sh +caplets add mcp ... +caplets add openapi ... +caplets add google-discovery ... +caplets add graphql ... +caplets add http ... +caplets add cli ... +``` + +Use [Caplet files](/reference/caplet-files/) for the resulting frontmatter and +[Configuration schema](/reference/config/) for JSON configuration. + +## Discover and call backends + +| Commands | Backend surface | +| -------------------------------------------------------------------------------- | ------------------------------------------------------ | +| `list-tools`, `search-tools`, `get-tool`, `call-tool` | Downstream tools and actions for every backend family. | +| `list-resources`, `search-resources`, `list-resource-templates`, `read-resource` | MCP resources and templates. | +| `list-prompts`, `search-prompts`, `get-prompt`, `complete` | MCP prompts and prompt/resource-template completion. | + +Qualified tool and prompt targets can use `.` where command help accepts a +`` argument. Prefer `--json` or `--format json` for scripts when supported. + +## Code Mode + +Run inline TypeScript or read it from a file: + +```sh +caplets code-mode 'return Object.keys(caplets)' +caplets code-mode --file ./query.ts +caplets code-mode types +``` + +The CLI invocation is one-shot. Live `sessionId` reuse belongs to a long-running MCP or native Code +Mode service; separate CLI processes do not share a QuickJS heap. + +## Serve, daemon, and attach + +| Command | Purpose | +| ------------------------------------------- | ------------------------------------------------------------------------------------- | +| `caplets serve` | Run configured Caplets as a foreground MCP stdio or HTTP server. | +| `caplets daemon install` | Install or update the managed local daemon. | +| `caplets daemon start` / `restart` / `stop` | Control the managed daemon. | +| `caplets daemon status` | Inspect daemon configuration and health. | +| `caplets daemon logs` | Read or follow daemon stdout/stderr. | +| `caplets daemon uninstall` | Remove the daemon; `--purge` also removes its config, state, and logs. | +| `caplets attach [url]` | Run a thin stdio MCP adapter backed by a local daemon, remote host, or Cloud runtime. | + +`attach` is stdio-only and does not open an HTTP listener. Use `serve --transport http +--upstream-url ` for a long-running stacked HTTP runtime. + +## Remote Login and Cloud + +Self-hosted Remote Login: + +```text +caplets remote login +caplets remote status [url] +caplets remote logout +caplets remote host clients +caplets remote host logins +caplets remote host approve --yes +caplets remote host deny +caplets remote host revoke +``` + +Host administration uses the SQL Authoritative Host State selected by user config. The deprecated +`--state-path` option is rejected. + +Hosted Cloud commands: + +```text +caplets cloud auth login +caplets cloud auth status +caplets cloud auth logout +caplets cloud auth workspaces +caplets cloud auth switch +caplets cloud add [path] +``` + +Saved Remote Profiles and Cloud credentials belong in Caplets client state, not agent config. + +## Vault and backend OAuth + +Vault commands: + +```text +caplets vault set +caplets vault get +caplets vault list +caplets vault delete +caplets vault access grant +caplets vault access list [name] [caplet-id] +caplets vault access revoke +``` + +Backend OAuth commands: + +```text +caplets auth login +caplets auth refresh +caplets auth list +caplets auth logout +``` + +Vault reveal is explicit (`vault get --show`). Metadata and access-list operations do not print raw +values. + +## Authoritative Host State + +```text +caplets storage status +caplets storage schema-migrate +caplets storage migrate-legacy +caplets storage records list --stored +caplets storage records get --stored +caplets storage records import +caplets storage records update +caplets storage records export +caplets storage records revisions +caplets storage records restore +caplets storage records delete-revision +caplets storage records retention +caplets storage records rename +caplets storage records delete +caplets storage records installation status +caplets storage records installation detach +caplets storage records installation observe +caplets storage records installation replace +``` + +Record mutations use required generation checks where command help shows `--generation`. Read +[Authoritative Host State](/storage/) before schema migration, legacy migration, or destructive +record administration. + +## Config, telemetry, and completion + +```text +caplets config path +caplets config paths +caplets telemetry status +caplets telemetry enable +caplets telemetry disable +caplets telemetry rotate-id +caplets telemetry delete-id +caplets telemetry debug -- +caplets completion +``` + +Load completion for the current shell session: + +```sh +source <(caplets completion bash) +source <(caplets completion zsh) +caplets completion fish | source +``` + +For PowerShell, run `caplets completion powershell | Invoke-Expression`. The `cmd` output defines a +DOSKEY completion macro. Persist the generated script using your shell's normal completion setup. + +## Environment selectors and controls + +| Variable | Effect | +| ------------------------------------ | ------------------------------------------------------------------------------- | +| `CAPLETS_CONFIG` | Override the user config path. | +| `CAPLETS_PROJECT_CONFIG` | Override the project config path. | +| `CAPLETS_MODE` | Select `auto`, `local`, `daemon`, `remote`, or `cloud` for native integrations. | +| `CAPLETS_DAEMON_URL` | Select the local loopback daemon for native mode. | +| `CAPLETS_REMOTE_URL` | Select a self-hosted or Cloud remote base URL. | +| `CAPLETS_REMOTE_WORKSPACE` | Select a workspace for a saved remote profile. | +| `CAPLETS_DISABLE_TELEMETRY=1` | Disable anonymous telemetry for the process. | +| `CAPLETS_DISABLE_UPDATE_CHECK=1` | Disable passive update metadata and notices. | +| `CAPLETS_DISABLE_CATALOG_INDEXING=1` | Disable best-effort public catalog indexing requests. | +| `CAPLETS_ENCRYPTION_KEY` | Supply the 32-byte base64url Vault encryption key. | + +User-config fields, command flags, and environment variables have command-specific precedence. See +[Configuration](/configuration/) and the selected command's help rather than assuming every flag has +an environment equivalent. diff --git a/apps/docs/src/content/docs/reference/config.mdx b/apps/docs/src/content/docs/reference/config.mdx index e14bd5d5..51f6fc84 100644 --- a/apps/docs/src/content/docs/reference/config.mdx +++ b/apps/docs/src/content/docs/reference/config.mdx @@ -98,6 +98,38 @@ Public OpenAPI endpoint: ## Major sections +### `storage` + +Authoritative Host State backend. SQLite is used when omitted. + +#### SQLite + +| Field | Status | Type | Description | +| -------------- | -------- | -------- | ------------------------------------------------------------------------ | +| `type` | Required | "sqlite" | Select SQLite Authoritative Host State. | +| `path` | Optional | string | SQLite database path. The platform state directory is used when omitted. | +| `assets` | Optional | object | SQL or S3-compatible Caplet bundle asset storage. | +| `bundleLimits` | Optional | object | Per-bundle auxiliary file count and byte limits. | + +#### PostgreSQL + +| Field | Status | Type | Description | +| ------------------ | -------- | ---------- | ------------------------------------------------- | +| `type` | Required | "postgres" | Select PostgreSQL Authoritative Host State. | +| `connectionString` | Required | string | PostgreSQL bootstrap connection URL. | +| `schema` | Optional | string | Logical-host schema; defaults to `caplets`. | +| `assets` | Optional | object | SQL or S3-compatible Caplet bundle asset storage. | +| `bundleLimits` | Optional | object | Per-bundle auxiliary file count and byte limits. | + +`assets.type` is `sql` or `s3`. S3 requires `region` and `bucket`; optional fields are +`endpoint`, `prefix`, `forcePathStyle`, `accessKeyId`, and `secretAccessKey`. Static access +keys must be configured together. Prefer workload identity or the AWS SDK credential chain. + +`bundleLimits.maxFiles` accepts integers from 1 through 100,000. `maxFileBytes` and +`maxTotalBytes` accept positive integers. +See [Authoritative Host State](/storage/) before PostgreSQL schema migration, S3 setup, backup, or +legacy migration. + ### `serve` User-owned HTTP serve defaults. Ignored from project config for security. diff --git a/apps/docs/src/content/docs/remote-attach.mdx b/apps/docs/src/content/docs/remote-attach.mdx index a6cc2520..f7537e93 100644 --- a/apps/docs/src/content/docs/remote-attach.mdx +++ b/apps/docs/src/content/docs/remote-attach.mdx @@ -14,7 +14,7 @@ upstream Caplets host. ## Modes -Caplets native integrations use the same mode names as `caplets attach`: +Caplets native integrations use these shared runtime selection modes: | Mode | Use it when | | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | @@ -72,8 +72,8 @@ caplets remote host logins caplets remote host approve --yes ``` -If the server uses a non-default credential state directory, pass the same directory -with `--state-path` that the server uses for `caplets serve --remote-state-path`. +Host approval commands use the Authoritative Host State backend from the host's user +config. The deprecated `--state-path` option no longer selects credential state. The visible approval code is not an attach credential, but do not put Caplets access or refresh credentials in shell history, agent configs, or environment variables. diff --git a/apps/docs/src/content/docs/storage.mdx b/apps/docs/src/content/docs/storage.mdx new file mode 100644 index 00000000..b99e6689 --- /dev/null +++ b/apps/docs/src/content/docs/storage.mdx @@ -0,0 +1,186 @@ +--- +title: Authoritative Host State +description: Configure, migrate, back up, and operate Caplets host state in SQLite or PostgreSQL. +--- + +Authoritative Host State is the correctness-bearing state owned by a Caplets host. It includes +stored Caplet records and revisions, installation provenance, backend OAuth state, encrypted Vault +values and access grants, remote clients and pending logins, setup state, dashboard sessions, +Project Binding coordination, Operator Activity, host identity, and schema metadata. + +One host uses exactly one SQL backend. Caplets does not dual-write, fall back to a legacy file store, +or copy between SQLite and PostgreSQL automatically. + +## SQLite: the default + +When `storage` is omitted, Caplets opens SQLite at: + +| Platform | Default database | +| --------------- | -------------------------------------------------------------------------------------------------- | +| Linux and macOS | Absolute `$XDG_STATE_HOME/caplets.sqlite3`; otherwise `$HOME/.local/state/caplets.sqlite3` | +| Windows | Absolute `%LOCALAPPDATA%\caplets.sqlite3`; otherwise `%USERPROFILE%\AppData\Local\caplets.sqlite3` | + +Override the path in the user config when the host needs a mounted or separately backed-up volume: + +```json +{ + "$schema": "https://caplets.dev/config.schema.json", + "storage": { + "type": "sqlite", + "path": "/srv/caplets/state/host.sqlite3" + } +} +``` + +`storage` is host-owned user configuration. Project config cannot override it. SQLite enables +foreign keys, WAL journaling, full synchronous writes, and a busy timeout. Ordered schema +migrations run before SQLite serves; a failed migration or a newer unknown schema fails closed. + +SQLite is the boring choice for one Host Node. Stop every process using the database before an +offline backup or restore. Include the configured S3 asset prefix in the same recovery point when +S3 asset storage is enabled. + +## PostgreSQL: multiple Host Nodes + +Use PostgreSQL when several Host Nodes must coordinate one logical host: + +```json +{ + "$schema": "https://caplets.dev/config.schema.json", + "storage": { + "type": "postgres", + "connectionString": "postgresql://caplets_runtime:@db.example:5432/caplets", + "schema": "caplets_prod" + } +} +``` + +One PostgreSQL schema is one logical Caplets host. Every node for that host must use the same +schema, configuration, global Caplet Files, Vault encryption key, and asset backend. The schema name +defaults to `caplets` and must match `^[a-z_][a-z0-9_]{0,62}$`. + +The connection URL is a bootstrap secret: SQL must open before Caplets Vault is available. Render +the user config from a deployment secret or mount a secret-generated config file. Do not put the +URL in Caplets Vault. Caplets does not define a `CAPLETS_STORAGE_*` environment mapping. + +PostgreSQL runtime startup never applies DDL. Run migrations as a one-shot job with a +DDL-capable connection before starting DML-only runtime nodes: + +```sh +caplets storage schema-migrate +caplets storage status --json +``` + +A host that cannot read the released schema version, sees the wrong version, or cannot reach its +asset backend is not ready. Do not replace the Caplets readiness check with a TCP-only database +check. + +## Bundle asset storage + +Stored Caplet bundle assets are content-addressed in SQL by default. An S3-compatible backend is +optional for assets that must survive node loss or be readable from any node: + +```json +{ + "storage": { + "type": "postgres", + "connectionString": "postgresql://caplets_runtime:@db.example:5432/caplets", + "schema": "caplets_prod", + "assets": { + "type": "s3", + "endpoint": "https://objects.example.com", + "region": "us-east-1", + "bucket": "caplets-production", + "prefix": "host-a", + "forcePathStyle": false + } + } +} +``` + +`region` and `bucket` are required. `endpoint`, `prefix`, and `forcePathStyle` are optional. Prefer +workload identity or the AWS SDK credential chain. If static credentials are unavoidable, +`accessKeyId` and `secretAccessKey` must be configured together and injected as deployment +secrets; they cannot come from Caplets Vault. + +Object uploads are read-back and hash-verified before SQL references them. If a current record +needs an unavailable object, storage becomes unhealthy rather than substituting empty content. +Back up SQL and the matching object version set as one recovery point. + +## Inspect and administer storage + +Check backend readiness, schema version, domain counts, and asset health: + +```sh +caplets storage status --json +``` + +Stored Caplet records are revisioned operator-managed bundles. Start with: + +```sh +caplets storage records list --stored +caplets storage records get --stored +caplets storage records revisions +``` + +Import and export complete bundles: + +```sh +caplets storage records import ./my-caplet +caplets storage records export ./exported-caplet +``` + +Mutating record commands use observed generations to reject stale writes. Run +`caplets storage records --help` and the selected subcommand's `--help` before update, restore, +rename, retention, revision deletion, installation-lifecycle, or hard-delete operations. + +## Upgrade from 0.25.x or earlier + +Caplets 0.26.0 moved Authoritative Host State from legacy files into SQL. Migration is an explicit, +offline, one-time upgrade step. Fresh hosts that never ran 0.25.x or earlier do not need it. + +The migration imports tracked global Caplets and their installation provenance from the old +lockfile, plus backend auth, Vault values and grants, remote-client security state, setup state, and +Operator Activity. It moves the old global lockfile into the timestamped backup after commit. +Untracked global Caplet Files remain in place as authoritative overlays. + +1. Install the new CLI, but do not start an upgraded Host Node. +2. Stop every old Host Node and back up all legacy state roots, the target SQL backend, and any + object-store prefix. +3. Run the dry run in the same user and environment that own the host: + + ```sh + caplets storage migrate-legacy --dry-run + ``` + +4. Resolve every missing artifact, lockfile hash or provenance mismatch, ID collision, malformed + legacy record, Vault key problem, and target conflict. +5. Run the write migration, then check readiness: + + ```sh + caplets storage migrate-legacy + caplets storage status --json + ``` + +The platform global Caplets root and lockfile are defaults. Use `--caplets-root` or `--lockfile` +only when those two paths are nonstandard. If an error says a tracked Caplet is missing, the message +names both the expected artifact path and lockfile; restore the artifact or remove the stale +lockfile entry before rerunning the dry run. + +The write command acquires an exclusive migration lock, imports applicable domains in one SQL +transaction, verifies counts and content hashes, and only then moves migrated source files into a +timestamped backup. It copies the still-active file-layer Vault key into that backup. Record the +printed backup path. After cutover, legacy files are not a fallback. + +## Backup and recovery boundaries + +- Back up SQLite with a consistent SQLite backup while every Host Node is stopped. Preserve WAL and + SHM sidecars for diagnosis until the restore decision is final. +- Back up PostgreSQL with the deployment's tested database backup process and include schema, + migration metadata, and matching object versions. +- Never restore a database schema newer than the running Caplets release. +- Live Code Mode heaps, Project Binding workspace bytes, and node-local media artifacts are not + reconstructed from SQL. Clustered deployments need affinity or explicit rebind/recovery for + those node-local resources. +- Caplets has no built-in SQLite-to-PostgreSQL copier. Rehearse any external transfer against a + disposable target, migrate the target schema first, and verify all domains before cutover. diff --git a/apps/docs/src/content/docs/troubleshooting.mdx b/apps/docs/src/content/docs/troubleshooting.mdx index aa89a605..7f5aeab8 100644 --- a/apps/docs/src/content/docs/troubleshooting.mdx +++ b/apps/docs/src/content/docs/troubleshooting.mdx @@ -20,6 +20,27 @@ configured integrations. Fix the first failed check, then rerun the same command ## Symptom map +### Auth, Vault values, or remote clients disappeared after upgrading + +Expected symptom: after upgrading from `caplets@0.25.x` or earlier to `caplets@0.26.0` +or later, previously configured credentials, Vault values or grants, remote clients, +setup history, activity, or tracked global Caplets appear missing. + +Do not recreate or overwrite the missing state. Caplets 0.26.0 and later use SQL as the +authority and do not fall back to the legacy files. Stop every Caplets Host Node, then run +the required one-time migration: + +```sh +caplets storage migrate-legacy --dry-run +caplets storage migrate-legacy +caplets storage status --json +``` + +Resolve dry-run failures before the write migration and restart only after storage status +reports ready. If a tracked Caplet is missing, restore the exact path named in the error or +remove its stale lockfile entry, then rerun the dry run. See +[Authoritative Host State](/storage/#upgrade-from-025x-or-earlier). + ### Agent cannot see Caplets Expected symptom: the agent has no `caplets__code_mode` tool, or no Caplets MCP server is @@ -242,19 +263,20 @@ Keep MCP client configs on `caplets attach `. ### Attach URL or workspace is wrong -Expected symptom: the agent connects to a stale workspace, or the runtime reports that the -attach manifest is missing or stale. +Expected symptom: the agent connects to an unintended runtime or workspace. -Rerun the attach command from the current workspace and copy the fresh values into the -agent environment: +Inspect the effective runtime selection rather than copying tokens or endpoint internals into +agent configuration: ```sh -caplets attach +caplets remote status +caplets doctor --json +caplets attach --once ``` -Use the versioned attach URL printed by the runtime. Attach connects to `/v1/attach`. -Ordinary MCP clients that are not using attach connect to `/v1/mcp`. Do not reuse an older -`/ws/...` path from shell history or a previous docs page. +Correct `CAPLETS_MODE`, `CAPLETS_REMOTE_URL`, or `CAPLETS_REMOTE_WORKSPACE`, then restart the +agent. Attach uses the runtime's `/v1/attach` API. Ordinary MCP clients that bypass attach use +`/v1/mcp`; do not substitute one path for the other. ### Project Binding smoke check fails @@ -269,13 +291,15 @@ ls .caplets/config.json caplets doctor ``` -Run the agent from the project root, then retry the smallest Code Mode script that uses the -project Caplet. If the capability needs setup commands, run the commands listed in the -Caplet's `setup` metadata before asking the agent to call it. +Run the agent from the project root. Managed Project Binding currently runs through native Cloud +sessions and native loopback daemon sessions that provide project context; ordinary CLI +`caplets attach ` is only a thin MCP adapter, and self-hosted HTTPS native remotes do not +currently establish a managed binding. Use `caplets attach --once` only to validate the +manifest policy and WebSocket endpoint. If the capability needs setup commands, run those listed +in its `setup` metadata before asking the agent to call it. -For stacked runtimes, project-local Caplets come from the attach or native session's project -root. The long-running HTTP server startup directory is not used as the active project for -every client. +See [Project Binding](/project-binding/) for exclusion rules, size limits, leases, and +node-loss recovery. ## OSV no-auth smoke test diff --git a/apps/docs/src/content/docs/vault.mdx b/apps/docs/src/content/docs/vault.mdx index b464dfae..6a81b983 100644 --- a/apps/docs/src/content/docs/vault.mdx +++ b/apps/docs/src/content/docs/vault.mdx @@ -177,26 +177,30 @@ CAPLETS_MODE=cloud CAPLETS_REMOTE_URL=https://cloud.caplets.dev caplets vault li Remote Vault values belong to the selected runtime. Local Caplets do not read, mirror, or forward remote or Cloud Vault values. -## Encryption key +## Encryption and key storage -For local and self-hosted stores, Caplets writes encrypted values under the Caplets state -directory. On Linux and macOS the default local path is: +Vault values and access grants are part of [Authoritative Host State](/storage/) in SQLite +or PostgreSQL. Values are encrypted with AES-256-GCM before Caplets writes them to SQL. +The encryption key remains outside SQL under the Caplets state directory. On Linux and +macOS the default key file is: ```text -~/.local/state/caplets/vault +~/.local/state/caplets/vault/vault-key ``` -The Vault encryption key is minted automatically on first write and stored as a private -file with `0600` permissions where the platform supports POSIX permissions. You can -instead provide a 32-byte base64url key with `CAPLETS_ENCRYPTION_KEY`: +The file key is created automatically when a host runtime initializes Vault storage, or when a +Vault value is first written if that happens earlier. It is stored with `0600` permissions where +the platform supports POSIX permissions. You can instead provide a 32-byte base64url key with +`CAPLETS_ENCRYPTION_KEY`: ```sh CAPLETS_ENCRYPTION_KEY=... caplets vault list ``` -Use the environment key only when you need to provide the same key from another secret -manager or deployment system. If the encryption key is unavailable or has the wrong -permissions, Vault-backed Caplets are quarantined until the key source is fixed. +Use the environment key only when the same key comes from another secret manager or +deployment system. Every node sharing one PostgreSQL host must use the same key. If the +key is unavailable or invalid, Vault-backed Caplets are quarantined until the key source +is fixed. ## Diagnostics diff --git a/docs/operations/sql-authoritative-host-state.md b/docs/operations/sql-authoritative-host-state.md index 33beb220..a2439239 100644 --- a/docs/operations/sql-authoritative-host-state.md +++ b/docs/operations/sql-authoritative-host-state.md @@ -7,6 +7,12 @@ SQLite fallback, SQLite write buffer, dual write, or built-in backend copier. Run storage administration as a server-local operator and keep every Host Node stopped whenever a procedure below says the operation is offline. +> **Required for upgrades to `caplets@0.26.0` or later:** If the host ran +> `caplets@0.25.x` or earlier, complete the +> [offline legacy filesystem migration](#offline-legacy-filesystem-migration) before +> restarting the daemon, running `caplets setup`, or serving requests. Caplets does not +> automatically migrate or fall back to legacy Authoritative Host State. + ## SQLite ### Location and configuration @@ -306,32 +312,35 @@ storage can be degraded independently only when a configured local fallback is v ## Offline legacy filesystem migration -This migration moves tracked global Caplets and the old global lockfile into the selected SQL -backend. Untracked global Caplet Files remain in place as authoritative overlays. There is no dual -write and no destructive migration at first startup. +This migration moves tracked global Caplets, backend auth tokens, Vault values and grants, +remote-client security state, setup state, Operator Activity Log entries, and the old global +lockfile into the selected SQL backend. Untracked global Caplet Files remain in place as +authoritative overlays. There is no dual write and no destructive migration at first startup. + +> **This is a required, one-time upgrade step.** Skipping it can make existing backend +> credentials, Vault values and grants, remote clients, setup history, Operator Activity, +> and tracked global Caplets appear missing. Fresh hosts that never ran a version before +> 0.26.0 do not need it. 1. Stop every Host Node (or enter exclusive maintenance). -2. Back up the global Caplets root, old lockfile, selected SQL database, and object prefix. -3. Verify with a dry run using explicit paths: +2. Back up every legacy state root, the selected SQL database, and the object prefix. +3. Verify with a dry run. The command uses the platform global-config directory and global + lockfile path by default; use `--caplets-root` or `--lockfile` only for nonstandard paths: ```sh - caplets storage migrate-legacy \ - --caplets-root /srv/caplets/global \ - --lockfile /srv/caplets/caplets.lock.json \ - --dry-run + caplets storage migrate-legacy --dry-run ``` 4. Resolve every missing artifact, hash/provenance mismatch, ID collision, or validation failure. 5. Run the same command without `--dry-run`: ```sh - caplets storage migrate-legacy \ - --caplets-root /srv/caplets/global \ - --lockfile /srv/caplets/caplets.lock.json + caplets storage migrate-legacy ``` -The command takes an exclusive migration lock, imports transactionally, compares counts/content -hashes, and only after commit moves migrated files and the old lockfile into a timestamped backup. +The command takes an exclusive migration lock, imports all applicable domains transactionally, +compares counts/content hashes, and only after commit moves migrated files and the old lockfile into +a timestamped backup. A shared Vault encryption key remains active and is copied into the backup. Record the printed backup path. Restart only after `caplets storage status --json` is ready. After cutover the legacy stores are not a fallback. diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index f87c4ae0..5f76ffdd 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -6,6 +6,12 @@ - 7dbfb74: Add authoritative SQLite and PostgreSQL host storage, relational Caplet Records with Markdown import/export, and SQL-backed host administration across CLI, HTTP, and remote-control surfaces. + **Migration required:** Hosts that ran `caplets@0.25.x` or earlier must stop every + Caplets Host Node and run `caplets storage migrate-legacy --dry-run`, then + `caplets storage migrate-legacy`, before restarting the daemon, running `caplets setup`, + or serving requests. Version 0.26.0 and later do not automatically migrate or fall back + to legacy auth, Vault, remote-security, setup, activity, or tracked-Caplet state. + ### Patch Changes - Updated dependencies [7dbfb74] diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 395bc86d..d86be9cc 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -6,6 +6,12 @@ - 7dbfb74: Add authoritative SQLite and PostgreSQL host storage, relational Caplet Records with Markdown import/export, and SQL-backed host administration across CLI, HTTP, and remote-control surfaces. + **Migration required for `caplets` CLI users:** Hosts that ran `caplets@0.25.x` or + earlier must stop every Caplets Host Node and run + `caplets storage migrate-legacy --dry-run`, then `caplets storage migrate-legacy`, + before restarting. The SQL-backed runtime does not fall back to legacy Authoritative + Host State. + ## 0.35.0 ### Minor Changes diff --git a/packages/core/src/cli.ts b/packages/core/src/cli.ts index 64458c75..c4780d03 100644 --- a/packages/core/src/cli.ts +++ b/packages/core/src/cli.ts @@ -131,7 +131,13 @@ import { type DaemonOperationOptions, } from "./daemon"; import { resolveServeOptions, serveResolvedCaplets, type ServeOptions } from "./serve"; -import { defaultTelemetryStateDir } from "./config/paths"; +import { + defaultAuthDir, + defaultCacheBaseDir, + defaultCapletsLockfilePath, + defaultStateBaseDir, + defaultTelemetryStateDir, +} from "./config/paths"; import { appendBasePath } from "./server/options"; import { acknowledgeTelemetryAttributionClaim, @@ -4143,8 +4149,8 @@ function configureStorageCommands( storage .command("migrate-legacy") .description("Migrate verified legacy Host state into SQL storage.") - .requiredOption("--caplets-root ", "tracked global Caplet installation root") - .requiredOption("--lockfile ", "tracked global Caplet lockfile") + .option("--caplets-root ", "tracked global Caplet installation root") + .option("--lockfile ", "tracked global Caplet lockfile") .option("--dry-run", "verify migration inputs without writing or moving files") .option("--backup-root ", "timestamped backup destination") .option( @@ -4154,18 +4160,36 @@ function configureStorageCommands( ) .action( async (options: { - capletsRoot: string; - lockfile: string; + capletsRoot?: string; + lockfile?: string; dryRun?: boolean; backupRoot?: string; operatorClient: string; }) => { const configPath = context.configPath(); + const capletsRoot = options.capletsRoot + ? resolve(options.capletsRoot) + : resolve(resolveCapletsRoot(configPath)); + const lockfilePath = options.lockfile + ? resolve(options.lockfile) + : defaultCapletsLockfilePath(context.env); + const authDir = context.io.authDir ?? defaultAuthDir(context.env); + const legacyVaultRoot = context.io.authDir + ? join(authDir, "vault") + : join(defaultStateBaseDir(context.env), "caplets", "vault"); + const remoteSecurityDir = join(authDir, "remote-server"); const report = await migrateLegacyHostState({ storage: loadHostStorageConfig(configPath), - capletsRoot: resolve(options.capletsRoot), - lockfilePath: resolve(options.lockfile), + capletsRoot, + lockfilePath, operatorClientId: options.operatorClient, + backendAuthDir: authDir, + legacyVaultRoot, + legacyVaultEnv: context.env, + targetVaultRoot: legacyVaultRoot, + remoteSecurityDir, + setupStateDir: join(defaultCacheBaseDir(context.env), "caplets", "setup"), + operatorActivityDir: remoteSecurityDir, ...(options.backupRoot ? { backupRoot: options.backupRoot } : {}), dryRun: options.dryRun ?? false, }); diff --git a/packages/core/src/storage/legacy-migration.ts b/packages/core/src/storage/legacy-migration.ts index db89bf3d..b11bcba8 100644 --- a/packages/core/src/storage/legacy-migration.ts +++ b/packages/core/src/storage/legacy-migration.ts @@ -1,6 +1,8 @@ import { createHash, type Hash } from "node:crypto"; import { + constants as fsConstants, closeSync, + copyFileSync, existsSync, lstatSync, mkdirSync, @@ -28,7 +30,7 @@ import { FileVaultStore, type LegacyVaultMigrationSnapshot } from "../vault"; import type { ImportCapletBundleInput } from "./caplet-records"; import type { OperatorPrincipal } from "./installations"; import { createHostStorage, type HostStorage } from "./database"; -import type { LegacyRecordVaultGrantImport } from "./vault-grants"; +import type { LegacyVaultGrantImport } from "./vault-grants"; import type { HostDatabaseTransaction, HostStorageConfig, @@ -86,13 +88,14 @@ type PlannedArtifact = { type BackupMove = { source: string; target: string; + preserveSource?: boolean | undefined; }; type PlannedLegacyState = { artifacts: PlannedArtifact[]; backendAuth: LegacyOAuthTokenBundleSnapshot; vault: LegacyVaultMigrationSnapshot; - vaultGrants: LegacyRecordVaultGrantImport[]; + vaultGrants: LegacyVaultGrantImport[]; remoteSecurity: RemoteServerCredentialState; setup: LegacySetupMigrationSnapshot; operatorActivity: DashboardActivityEntry[]; @@ -141,6 +144,7 @@ export async function migrateLegacyHostState( ); if ( options.legacyVaultRoot !== undefined && + options.targetVaultRoot === undefined && resolve(storage.vaultValues.root) === resolve(options.legacyVaultRoot) ) { throw new CapletsError( @@ -183,12 +187,10 @@ export async function migrateLegacyHostState( await storage.remoteSecurity.assertLegacySnapshotImportable(plan.remoteSecurity); await storage.setupState.assertLegacySnapshotImportable(plan.setup); await storage.operatorActivity.assertLegacyEntriesImportable(plan.operatorActivity); - const existingVaultGrants = await assertRequiredVaultRecords( - storage, - plan.vaultGrants, - plan.artifacts, + await storage.vaultGrants.assertLegacyGrantsImportable( + await vaultGrantsAvailableBeforeImport(storage, plan.vaultGrants), + operator, ); - await storage.vaultGrants.assertLegacyRecordGrantsImportable(existingVaultGrants, operator); if (options.dryRun) { return reportForPlan("verified", null, plan); @@ -255,7 +257,7 @@ function importAndVerifyLegacySqlite( ); runSynchronous(storage.vaultValues.importLegacyValuesInTransaction(plan.vault.values, adapter)); runSynchronous( - storage.vaultGrants.importLegacyRecordGrantsInTransaction(plan.vaultGrants, operator, adapter), + storage.vaultGrants.importLegacyGrantsInTransaction(plan.vaultGrants, operator, adapter), ); runSynchronous( storage.remoteSecurity.importLegacySnapshotInTransaction(plan.remoteSecurity, adapter), @@ -271,7 +273,7 @@ function importAndVerifyLegacySqlite( ); runSynchronous(storage.vaultValues.verifyLegacyValuesInTransaction(plan.vault.values, adapter)); runSynchronous( - storage.vaultGrants.verifyLegacyRecordGrantsInTransaction(plan.vaultGrants, operator, adapter), + storage.vaultGrants.verifyLegacyGrantsInTransaction(plan.vaultGrants, operator, adapter), ); runSynchronous( storage.remoteSecurity.verifyLegacySnapshotInTransaction(plan.remoteSecurity, adapter), @@ -293,11 +295,7 @@ async function importAndVerifyLegacyPostgres( await storage.caplets.importBundlesInTransaction(pendingBundles, adapter); await storage.backendAuth.importLegacyBundlesInTransaction(plan.backendAuth.bundles, adapter); await storage.vaultValues.importLegacyValuesInTransaction(plan.vault.values, adapter); - await storage.vaultGrants.importLegacyRecordGrantsInTransaction( - plan.vaultGrants, - operator, - adapter, - ); + await storage.vaultGrants.importLegacyGrantsInTransaction(plan.vaultGrants, operator, adapter); await storage.remoteSecurity.importLegacySnapshotInTransaction(plan.remoteSecurity, adapter); await storage.setupState.importLegacySnapshotInTransaction(plan.setup, adapter); await storage.operatorActivity.importLegacyEntriesInTransaction(plan.operatorActivity, adapter); @@ -305,11 +303,7 @@ async function importAndVerifyLegacyPostgres( await verifyImportedArtifactsPostgres(storage, plan.artifacts, adapter); await storage.backendAuth.verifyLegacyBundlesInTransaction(plan.backendAuth.bundles, adapter); await storage.vaultValues.verifyLegacyValuesInTransaction(plan.vault.values, adapter); - await storage.vaultGrants.verifyLegacyRecordGrantsInTransaction( - plan.vaultGrants, - operator, - adapter, - ); + await storage.vaultGrants.verifyLegacyGrantsInTransaction(plan.vaultGrants, operator, adapter); await storage.remoteSecurity.verifyLegacySnapshotInTransaction(plan.remoteSecurity, adapter); await storage.setupState.verifyLegacySnapshotInTransaction(plan.setup, adapter); await storage.operatorActivity.verifyLegacyEntriesInTransaction(plan.operatorActivity, adapter); @@ -384,9 +378,12 @@ function planLegacyState(options: LegacyMigrationOptions, backupPath: string): P vault.snapshot.sourcePaths, options.legacyVaultRoot, join(backupPath, "vault"), + resolve(options.targetVaultRoot ?? "") === resolve(options.legacyVaultRoot) + ? resolve(join(options.legacyVaultRoot, "vault-key")) + : undefined, ); } - const vaultGrants = plannedVaultGrants(vault.snapshot); + const vaultGrants = plannedVaultGrants(vault.snapshot, artifacts); const remoteSecurity = options.remoteSecurityDir ? snapshotWithBackupFallback( @@ -487,56 +484,57 @@ function appendFileBackupMoves( sourcePaths: string[], sourceRoot: string, backupRoot: string, + preservedSource?: string, ): void { const resolvedRoot = resolve(sourceRoot); for (const source of sourcePaths) { + const resolvedSource = resolve(source); moves.push({ source, - target: join(backupRoot, portableRelative(resolvedRoot, resolve(source))), + target: join(backupRoot, portableRelative(resolvedRoot, resolvedSource)), + ...(preservedSource === resolvedSource ? { preserveSource: true } : {}), }); } } function plannedVaultGrants( snapshot: LegacyVaultMigrationSnapshot, -): LegacyRecordVaultGrantImport[] { + artifacts: PlannedArtifact[], +): LegacyVaultGrantImport[] { + const trackedGlobalIds = new Set(artifacts.map((artifact) => artifact.id)); const identities = new Set(); return snapshot.grants.map((grant) => { - const identity = JSON.stringify([grant.capletId, grant.referenceName]); + const storedRecord = + grant.origin.kind === "global-file" && trackedGlobalIds.has(grant.capletId); + const originKind = storedRecord ? "stored-record" : grant.origin.kind; + const originPath = storedRecord ? null : grant.origin.path; + const identity = JSON.stringify([grant.capletId, originKind, originPath, grant.referenceName]); if (identities.has(identity)) { - throw new CapletsError( - "CONFIG_INVALID", - "Legacy Vault grants cannot be bound uniquely to immutable Caplet Records.", - ); + throw new CapletsError("CONFIG_INVALID", "Legacy Vault grants contain duplicate subjects."); } identities.add(identity); return { capletId: grant.capletId, vaultKey: grant.storedKey, referenceName: grant.referenceName, + originKind, + originPath, createdAt: grant.createdAt, }; }); } -async function assertRequiredVaultRecords( +async function vaultGrantsAvailableBeforeImport( storage: HostStorage, - grants: LegacyRecordVaultGrantImport[], - artifacts: PlannedArtifact[], -): Promise { - const plannedIds = new Set(artifacts.map((artifact) => artifact.id)); - const existing: LegacyRecordVaultGrantImport[] = []; + grants: LegacyVaultGrantImport[], +): Promise { + const available: LegacyVaultGrantImport[] = []; for (const grant of grants) { - const record = await storage.caplets.get(grant.capletId); - if (!record && !plannedIds.has(grant.capletId)) { - throw new CapletsError( - "CONFIG_INVALID", - `Caplet Record ${grant.capletId} required by a legacy Vault grant was not found.`, - ); + if (grant.originKind !== "stored-record" || (await storage.caplets.get(grant.capletId))) { + available.push(grant); } - if (record) existing.push(grant); } - return existing; + return available; } function reportForPlan( @@ -573,7 +571,10 @@ function planArtifacts(capletsRoot: string, lockfilePath: string): PlannedArtifa return lockfile.entries.map((entry) => { const destination = validateLockfileDestination(capletsRoot, entry.destination); if (!existsSync(destination)) { - throw new CapletsError("CONFIG_NOT_FOUND", `Tracked Caplet ${entry.id} is missing.`); + throw new CapletsError( + "CONFIG_NOT_FOUND", + `Tracked Caplet ${entry.id} is missing at ${destination}. Restore it or remove its stale entry from ${lockfilePath} before migration.`, + ); } const stats = lstatSync(destination); if ( @@ -747,18 +748,26 @@ function moveLegacyArtifactsToBackup(moves: BackupMove[]): void { uniqueTargets.add(target); } - const moved: BackupMove[] = []; + const completed: BackupMove[] = []; try { for (const move of moves) { mkdirSync(dirname(move.target), { recursive: true, mode: 0o700 }); - renameSync(move.source, move.target); - moved.push(move); + if (move.preserveSource) { + copyFileSync(move.source, move.target, fsConstants.COPYFILE_EXCL); + } else { + renameSync(move.source, move.target); + } + completed.push(move); } } catch (error) { - for (const entry of moved.reverse()) { + for (const entry of completed.reverse()) { if (!existsSync(entry.target)) continue; - mkdirSync(dirname(entry.source), { recursive: true, mode: 0o700 }); - renameSync(entry.target, entry.source); + if (entry.preserveSource) { + rmSync(entry.target, { force: true }); + } else { + mkdirSync(dirname(entry.source), { recursive: true, mode: 0o700 }); + renameSync(entry.target, entry.source); + } } throw error; } diff --git a/packages/core/src/storage/vault-grants.ts b/packages/core/src/storage/vault-grants.ts index 458d6dda..9fb9d483 100644 --- a/packages/core/src/storage/vault-grants.ts +++ b/packages/core/src/storage/vault-grants.ts @@ -33,10 +33,12 @@ export type VaultGrantInput = { operator: OperatorPrincipal; }; -export type LegacyRecordVaultGrantImport = { +export type LegacyVaultGrantImport = { capletId: string; vaultKey: string; referenceName: string; + originKind: ConfigSourceKind; + originPath: string | null; createdAt: string; }; @@ -100,87 +102,85 @@ export class VaultGrantStore { : await revokePostgres(this.database.db, normalized, operatorId); } - async assertLegacyRecordGrantsImportable( - grants: LegacyRecordVaultGrantImport[], + async assertLegacyGrantsImportable( + grants: LegacyVaultGrantImport[], operator: OperatorPrincipal, ): Promise { const operatorId = requireOperator(operator); - const validated = validateLegacyRecordGrantImports(grants); + const validated = validateLegacyGrantImports(grants); if (this.database.dialect === "sqlite") { - assertLegacyRecordGrantsMatchSqlite(this.database.db, validated, operatorId); + assertLegacyGrantsMatchSqlite(this.database.db, validated, operatorId); } else { - await assertLegacyRecordGrantsMatchPostgres(this.database.db, validated, operatorId); + await assertLegacyGrantsMatchPostgres(this.database.db, validated, operatorId); } } - async importLegacyRecordGrants( - grants: LegacyRecordVaultGrantImport[], + async importLegacyGrants( + grants: LegacyVaultGrantImport[], operator: OperatorPrincipal, ): Promise { const operatorId = requireOperator(operator); - const validated = validateLegacyRecordGrantImports(grants); + const validated = validateLegacyGrantImports(grants); if (validated.length === 0) return; if (this.database.dialect === "sqlite") { this.database.db.transaction((transaction) => - importLegacyRecordGrantsSqlite(transaction, validated, operatorId), + importLegacyGrantsSqlite(transaction, validated, operatorId), ); return; } await this.database.db.transaction( - async (transaction) => - await importLegacyRecordGrantsPostgres(transaction, validated, operatorId), + async (transaction) => await importLegacyGrantsPostgres(transaction, validated, operatorId), ); } - importLegacyRecordGrantsInTransaction( - grants: LegacyRecordVaultGrantImport[], + importLegacyGrantsInTransaction( + grants: LegacyVaultGrantImport[], operator: OperatorPrincipal, transaction: HostDatabaseTransaction, ): void | Promise { const operatorId = requireOperator(operator); - const validated = validateLegacyRecordGrantImports(grants); + const validated = validateLegacyGrantImports(grants); if (validated.length === 0) return; return transaction.dialect === "sqlite" - ? importLegacyRecordGrantsSqlite(transaction.db, validated, operatorId) - : importLegacyRecordGrantsPostgres(transaction.db, validated, operatorId); + ? importLegacyGrantsSqlite(transaction.db, validated, operatorId) + : importLegacyGrantsPostgres(transaction.db, validated, operatorId); } - async verifyLegacyRecordGrants( - grants: LegacyRecordVaultGrantImport[], + async verifyLegacyGrants( + grants: LegacyVaultGrantImport[], operator: OperatorPrincipal, ): Promise { const operatorId = requireOperator(operator); const stored = await this.list(); - for (const grant of validateLegacyRecordGrantImports(grants)) { + for (const grant of validateLegacyGrantImports(grants)) { const match = stored.find( (candidate) => - candidate.subjectKind === "record" && candidate.capletId === grant.capletId && candidate.vaultKey === grant.vaultKey && candidate.referenceName === grant.referenceName && - candidate.originKind === "stored-record" && - candidate.originPath === null && + candidate.originKind === grant.originKind && + candidate.originPath === grant.originPath && candidate.createdAt === grant.createdAt && candidate.createdBy === operatorId, ); if (!match) { throw new CapletsError( "INTERNAL_ERROR", - `Vault grant for Caplet Record ${grant.capletId} failed post-migration verification.`, + `Vault grant for ${grant.capletId} failed post-migration verification.`, ); } } } - verifyLegacyRecordGrantsInTransaction( - grants: LegacyRecordVaultGrantImport[], + verifyLegacyGrantsInTransaction( + grants: LegacyVaultGrantImport[], operator: OperatorPrincipal, transaction: HostDatabaseTransaction, ): void | Promise { const operatorId = requireOperator(operator); - const validated = validateLegacyRecordGrantImports(grants); + const validated = validateLegacyGrantImports(grants); return transaction.dialect === "sqlite" - ? verifyLegacyRecordGrantsSqlite(transaction.db, validated, operatorId) - : verifyLegacyRecordGrantsPostgres(transaction.db, validated, operatorId); + ? verifyLegacyGrantsSqlite(transaction.db, validated, operatorId) + : verifyLegacyGrantsPostgres(transaction.db, validated, operatorId); } async list(capletId?: string): Promise { @@ -239,12 +239,12 @@ type SqliteVaultGrantDatabase = type PostgresVaultGrantDatabase = | PostgresHostDatabase | Parameters[0]>[0]; -function importLegacyRecordGrantsSqlite( +function importLegacyGrantsSqlite( database: SqliteVaultGrantDatabase, - grants: LegacyRecordVaultGrantImport[], + grants: LegacyVaultGrantImport[], operatorId: string, ): void { - const rows = assertLegacyRecordGrantsMatchSqlite(database, grants, operatorId); + const rows = assertLegacyGrantsMatchSqlite(database, grants, operatorId); const pending = rows.filter((row) => row.existing === undefined); if (pending.length > 0) { database @@ -254,9 +254,9 @@ function importLegacyRecordGrantsSqlite( } } -async function importLegacyRecordGrantsPostgres( +async function importLegacyGrantsPostgres( database: PostgresVaultGrantDatabase, - grants: LegacyRecordVaultGrantImport[], + grants: LegacyVaultGrantImport[], operatorId: string, ): Promise { for (const grant of grants) { @@ -264,50 +264,61 @@ async function importLegacyRecordGrantsPostgres( sql`select pg_advisory_xact_lock(hashtextextended(${JSON.stringify([ "vault-legacy-grant", grant.capletId, + grant.originKind, + grant.originPath, grant.referenceName, ])}, 0))`, ); } - const rows = await assertLegacyRecordGrantsMatchPostgres(database, grants, operatorId); + const rows = await assertLegacyGrantsMatchPostgres(database, grants, operatorId); const pending = rows.filter((row) => row.existing === undefined); if (pending.length > 0) { await database.insert(postgres.vaultAccessGrants).values(pending.map((row) => row.values)); } } -function verifyLegacyRecordGrantsSqlite( +function verifyLegacyGrantsSqlite( database: SqliteVaultGrantDatabase, - grants: LegacyRecordVaultGrantImport[], + grants: LegacyVaultGrantImport[], operatorId: string, ): void { - const rows = assertLegacyRecordGrantsMatchSqlite(database, grants, operatorId); + const rows = assertLegacyGrantsMatchSqlite(database, grants, operatorId); if (rows.some((row) => row.existing === undefined)) { throw new CapletsError("INTERNAL_ERROR", "A Vault grant failed post-migration verification."); } } -async function verifyLegacyRecordGrantsPostgres( +async function verifyLegacyGrantsPostgres( database: PostgresVaultGrantDatabase, - grants: LegacyRecordVaultGrantImport[], + grants: LegacyVaultGrantImport[], operatorId: string, ): Promise { - const rows = await assertLegacyRecordGrantsMatchPostgres(database, grants, operatorId); + const rows = await assertLegacyGrantsMatchPostgres(database, grants, operatorId); if (rows.some((row) => row.existing === undefined)) { throw new CapletsError("INTERNAL_ERROR", "A Vault grant failed post-migration verification."); } } -function validateLegacyRecordGrantImports( - grants: LegacyRecordVaultGrantImport[], -): LegacyRecordVaultGrantImport[] { +function validateLegacyGrantImports(grants: LegacyVaultGrantImport[]): LegacyVaultGrantImport[] { const identities = new Set(); const validated = grants.map((grant) => { validateCapletId(grant.capletId); const vaultKey = validateGrantName(grant.vaultKey); const referenceName = validateGrantName(grant.referenceName); - const identity = JSON.stringify([grant.capletId, referenceName]); + const originValid = + Object.hasOwn(CONFIG_SOURCE_KINDS, grant.originKind) && + (grant.originKind === "stored-record" + ? grant.originPath === null + : typeof grant.originPath === "string" && grant.originPath.length > 0); + const identity = JSON.stringify([ + grant.capletId, + grant.originKind, + grant.originPath, + referenceName, + ]); if ( identities.has(identity) || + !originValid || !Number.isFinite(Date.parse(grant.createdAt)) || new Date(grant.createdAt).toISOString() !== grant.createdAt ) { @@ -319,83 +330,65 @@ function validateLegacyRecordGrantImports( return validated.sort( (left, right) => left.capletId.localeCompare(right.capletId) || + left.originKind.localeCompare(right.originKind) || + (left.originPath ?? "").localeCompare(right.originPath ?? "") || left.referenceName.localeCompare(right.referenceName), ); } -function assertLegacyRecordGrantsMatchSqlite( +function assertLegacyGrantsMatchSqlite( database: SqliteVaultGrantDatabase, - grants: LegacyRecordVaultGrantImport[], + grants: LegacyVaultGrantImport[], operatorId: string, ) { return grants.map((grant) => { - const recordKey = database - .select({ recordKey: sqlite.capletRecords.recordKey }) - .from(sqlite.capletRecords) - .where(eq(sqlite.capletRecords.capletId, grant.capletId)) - .get()?.recordKey; - if (!recordKey) { - throw new CapletsError( - "CONFIG_INVALID", - `Caplet Record ${grant.capletId} required by a legacy Vault grant was not found.`, - ); - } - const values = legacyRecordGrantValues(recordKey, grant, operatorId); + const subject = legacyGrantSubjectSqlite(database, grant); + const values = legacyGrantValues(subject, grant, operatorId); const existing = database .select() .from(sqlite.vaultAccessGrants) .where( and( - eq(sqlite.vaultAccessGrants.subjectKind, "record"), - eq(sqlite.vaultAccessGrants.subjectKey, recordKey), + eq(sqlite.vaultAccessGrants.subjectKind, subject.kind), + eq(sqlite.vaultAccessGrants.subjectKey, subject.key), eq(sqlite.vaultAccessGrants.referenceName, grant.referenceName), ), ) .get(); - if (existing && !legacyRecordGrantMatches(existing, values)) { + if (existing && !legacyGrantMatches(existing, values)) { throw new CapletsError( "CONFIG_EXISTS", - `Vault grant for Caplet Record ${grant.capletId} conflicts with the legacy snapshot.`, + `Vault grant for ${grant.capletId} conflicts with the legacy snapshot.`, ); } return { existing, values }; }); } -async function assertLegacyRecordGrantsMatchPostgres( +async function assertLegacyGrantsMatchPostgres( database: PostgresVaultGrantDatabase, - grants: LegacyRecordVaultGrantImport[], + grants: LegacyVaultGrantImport[], operatorId: string, ) { const rows = []; for (const grant of grants) { - const [record] = await database - .select({ recordKey: postgres.capletRecords.recordKey }) - .from(postgres.capletRecords) - .where(eq(postgres.capletRecords.capletId, grant.capletId)) - .limit(1); - if (!record) { - throw new CapletsError( - "CONFIG_INVALID", - `Caplet Record ${grant.capletId} required by a legacy Vault grant was not found.`, - ); - } - const values = legacyRecordGrantValues(record.recordKey, grant, operatorId); + const subject = await legacyGrantSubjectPostgres(database, grant); + const values = legacyGrantValues(subject, grant, operatorId); const [existing] = await database .select() .from(postgres.vaultAccessGrants) .where( and( - eq(postgres.vaultAccessGrants.subjectKind, "record"), - eq(postgres.vaultAccessGrants.subjectKey, record.recordKey), + eq(postgres.vaultAccessGrants.subjectKind, subject.kind), + eq(postgres.vaultAccessGrants.subjectKey, subject.key), eq(postgres.vaultAccessGrants.referenceName, grant.referenceName), ), ) .limit(1); - if (existing && !legacyRecordGrantMatches(existing, values)) { + if (existing && !legacyGrantMatches(existing, values)) { throw new CapletsError( "CONFIG_EXISTS", - `Vault grant for Caplet Record ${grant.capletId} conflicts with the legacy snapshot.`, + `Vault grant for ${grant.capletId} conflicts with the legacy snapshot.`, ); } rows.push({ existing, values }); @@ -403,48 +396,93 @@ async function assertLegacyRecordGrantsMatchPostgres( return rows; } -type LegacyRecordGrantRow = { - subjectKind: "record"; +function legacyGrantSubjectSqlite( + database: SqliteVaultGrantDatabase, + grant: LegacyVaultGrantImport, +): VaultGrantSubject { + if (grant.originKind !== "stored-record") { + return fileSubject(grant.capletId, grant.originKind, grant.originPath as string); + } + const recordKey = database + .select({ recordKey: sqlite.capletRecords.recordKey }) + .from(sqlite.capletRecords) + .where(eq(sqlite.capletRecords.capletId, grant.capletId)) + .get()?.recordKey; + if (!recordKey) { + throw new CapletsError( + "CONFIG_INVALID", + `Caplet Record ${grant.capletId} required by a legacy Vault grant was not found.`, + ); + } + return { kind: "record", key: recordKey, recordKey, capletId: null }; +} + +async function legacyGrantSubjectPostgres( + database: PostgresVaultGrantDatabase, + grant: LegacyVaultGrantImport, +): Promise { + if (grant.originKind !== "stored-record") { + return fileSubject(grant.capletId, grant.originKind, grant.originPath as string); + } + const [record] = await database + .select({ recordKey: postgres.capletRecords.recordKey }) + .from(postgres.capletRecords) + .where(eq(postgres.capletRecords.capletId, grant.capletId)) + .limit(1); + if (!record) { + throw new CapletsError( + "CONFIG_INVALID", + `Caplet Record ${grant.capletId} required by a legacy Vault grant was not found.`, + ); + } + return { kind: "record", key: record.recordKey, recordKey: record.recordKey, capletId: null }; +} + +type LegacyGrantRow = { + subjectKind: "record" | "file"; subjectKey: string; - recordKey: string; - capletId: null; + recordKey: string | null; + capletId: string | null; vaultKey: string; referenceName: string; - originKind: "stored-record"; - originPath: null; + originKind: ConfigSourceKind; + originPath: string | null; createdAt: string; createdBy: string; }; -function legacyRecordGrantValues( - recordKey: string, - grant: LegacyRecordVaultGrantImport, +function legacyGrantValues( + subject: VaultGrantSubject, + grant: LegacyVaultGrantImport, operatorId: string, -): LegacyRecordGrantRow { +): LegacyGrantRow { return { - subjectKind: "record" as const, - subjectKey: recordKey, - recordKey, - capletId: null, + subjectKind: subject.kind, + subjectKey: subject.key, + recordKey: subject.recordKey, + capletId: subject.capletId, vaultKey: grant.vaultKey, referenceName: grant.referenceName, - originKind: "stored-record" as const, - originPath: null, + originKind: grant.originKind, + originPath: grant.originPath, createdAt: grant.createdAt, createdBy: operatorId, }; } -function legacyRecordGrantMatches( +function legacyGrantMatches( existing: typeof sqlite.vaultAccessGrants.$inferSelect, - expected: LegacyRecordGrantRow, + expected: LegacyGrantRow, ): boolean { return ( + existing.subjectKind === expected.subjectKind && + existing.subjectKey === expected.subjectKey && existing.recordKey === expected.recordKey && - existing.capletId === null && + existing.capletId === expected.capletId && existing.vaultKey === expected.vaultKey && - existing.originKind === "stored-record" && - existing.originPath === null && + existing.referenceName === expected.referenceName && + existing.originKind === expected.originKind && + existing.originPath === expected.originPath && existing.createdAt === expected.createdAt && existing.createdBy === expected.createdBy ); diff --git a/packages/core/test/legacy-migration.test.ts b/packages/core/test/legacy-migration.test.ts index 161d1610..79f76b86 100644 --- a/packages/core/test/legacy-migration.test.ts +++ b/packages/core/test/legacy-migration.test.ts @@ -292,6 +292,36 @@ describe("legacy host-state migration", () => { } }); + it("identifies the missing tracked path and stale lockfile during migration", async () => { + const root = mkdtempSync(join(tmpdir(), "caplets-legacy-migration-missing-")); + directories.push(root); + const repository = join(root, "repository"); + const source = join(repository, "caplets", "sample"); + const capletsRoot = join(root, "host", "caplets"); + const lockfilePath = join(root, "host", "caplets.lock.json"); + const destination = join(capletsRoot, "sample"); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, "CAPLET.md"), document()); + installCaplets(repository, { + capletIds: ["sample"], + destinationRoot: capletsRoot, + lockfilePath, + }); + rmSync(destination, { recursive: true, force: true }); + + await expect( + migrateLegacyHostState({ + storage: { type: "sqlite", path: join(root, "host", "caplets.sqlite3") }, + capletsRoot, + lockfilePath, + operatorClientId: "operator_migration", + }), + ).rejects.toMatchObject({ + code: "CONFIG_NOT_FOUND", + message: `Tracked Caplet sample is missing at ${destination}. Restore it or remove its stale entry from ${lockfilePath} before migration.`, + }); + }); + it("dry-runs every applicable legacy domain without SQL state writes or source moves", async () => { const fixture = await createFullLegacyFixture(); const report = await migrateLegacyHostState({ ...fixture.options, dryRun: true }); diff --git a/packages/core/test/storage-records-cli.test.ts b/packages/core/test/storage-records-cli.test.ts index cf847501..a9972bb7 100644 --- a/packages/core/test/storage-records-cli.test.ts +++ b/packages/core/test/storage-records-cli.test.ts @@ -2,6 +2,8 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { writeTokenBundle } from "../src/auth/store"; +import { FileVaultStore } from "../src/vault"; import { runCli } from "../src/cli"; import { installCaplets } from "../src/install"; import { hostConfigGenerations } from "../src/storage/schema/sqlite"; @@ -254,15 +256,25 @@ describe("stored Caplet Record CLI", () => { } }); - it("requires explicit legacy Caplet paths for storage migrate-legacy", async () => { + it("uses platform paths to migrate standard legacy Host state by default", async () => { const root = mkdtempSync(join(tmpdir(), "caplets-storage-migrate-legacy-cli-")); directories.push(root); const repository = join(root, "repository"); const source = join(repository, "caplets", "legacy-cli"); - const capletsRoot = join(root, "legacy-host", "caplets"); - const lockfilePath = join(root, "legacy-host", "caplets.lock.json"); - const databasePath = join(root, "state", "caplets.sqlite3"); - const configPath = join(root, "config.json"); + const capletsRoot = join(root, "config", "caplets"); + const stateBase = join(root, "legacy-state"); + const lockfilePath = join(stateBase, "caplets", "caplets.lock.json"); + const cacheBase = join(root, "legacy-cache"); + const authDir = join(stateBase, "caplets", "auth"); + const vaultRoot = join(stateBase, "caplets", "vault"); + const databasePath = join(root, "sql", "caplets.sqlite3"); + const configPath = join(capletsRoot, "config.json"); + const env = { + CAPLETS_CONFIG: configPath, + XDG_STATE_HOME: stateBase, + XDG_CACHE_HOME: cacheBase, + }; + mkdirSync(capletsRoot, { recursive: true }); writeFileSync(configPath, JSON.stringify({ storage: { type: "sqlite", path: databasePath } })); writeBundle(source, "legacy-command", "Legacy"); installCaplets(repository, { @@ -270,31 +282,111 @@ describe("stored Caplet Record CLI", () => { destinationRoot: capletsRoot, lockfilePath, }); - - const output: string[] = []; - await runCli( - [ - "storage", - "migrate-legacy", - "--caplets-root", - capletsRoot, - "--lockfile", - lockfilePath, - "--dry-run", - ], + const overlayPath = join(capletsRoot, "overlay.md"); + writeFileSync(overlayPath, caplet("overlay-command", "Overlay")); + writeTokenBundle( { - env: { CAPLETS_CONFIG: configPath }, - writeOut: (value) => output.push(value), + server: "legacy-cli", + authType: "oauth2", + accessToken: "legacy-access-token", }, + authDir, ); + const vault = new FileVaultStore({ root: vaultRoot, env }); + vault.set("LEGACY_TOKEN", "legacy-vault-value"); + vault.grantAccess({ + storedKey: "LEGACY_TOKEN", + referenceName: "TOKEN", + capletId: "legacy-cli", + origin: { + kind: "global-file", + path: join(capletsRoot, "legacy-cli", "CAPLET.md"), + }, + }); + vault.grantAccess({ + storedKey: "LEGACY_TOKEN", + referenceName: "TOKEN", + capletId: "overlay", + origin: { kind: "global-file", path: overlayPath }, + }); + + const expectedDomains = { + backendAuthTokenBundles: 1, + vaultValues: 1, + vaultGrants: 2, + remotePairingCodes: 0, + remoteClients: 0, + remotePendingLogins: 0, + setupApprovals: 0, + setupAttempts: 0, + operatorActivityEntries: 0, + dashboardSessions: "not_applicable_no_legacy_format", + projectBindings: "not_applicable_no_legacy_format", + }; + + const output: string[] = []; + await runCli(["storage", "migrate-legacy", "--dry-run"], { + env, + writeOut: (value) => output.push(value), + }); expect(JSON.parse(output.join(""))).toEqual({ status: "verified", records: 1, installations: 1, backupPath: null, + domains: expectedDomains, }); expect(existsSync(join(capletsRoot, "legacy-cli", "CAPLET.md"))).toBe(true); expect(existsSync(lockfilePath)).toBe(true); + + const migratedOutput: string[] = []; + await runCli(["storage", "migrate-legacy"], { + env, + writeOut: (value) => migratedOutput.push(value), + }); + const migrated = JSON.parse(migratedOutput.join("")) as { + backupPath: string; + }; + expect(migrated).toEqual({ + status: "migrated", + records: 1, + installations: 1, + backupPath: expect.any(String), + domains: expectedDomains, + }); + expect(existsSync(join(authDir, "legacy-cli.json"))).toBe(false); + expect(existsSync(vault.valuePath("LEGACY_TOKEN"))).toBe(false); + expect(existsSync(vault.paths.keyFile)).toBe(true); + expect(existsSync(join(migrated.backupPath, "vault", "vault-key"))).toBe(true); + expect(existsSync(overlayPath)).toBe(true); + + const storage = await createHostStorage({ type: "sqlite", path: databasePath }, { vaultRoot }); + try { + await expect(storage.backendAuth.readTokenBundle("legacy-cli")).resolves.toMatchObject({ + bundle: { accessToken: "legacy-access-token" }, + }); + await expect(storage.vaultValues.resolveValue("LEGACY_TOKEN")).resolves.toBe( + "legacy-vault-value", + ); + await expect(storage.vaultGrants.list()).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + subjectKind: "record", + capletId: "legacy-cli", + originKind: "stored-record", + originPath: null, + }), + expect.objectContaining({ + subjectKind: "file", + capletId: "overlay", + originKind: "global-file", + originPath: overlayPath, + }), + ]), + ); + } finally { + await storage.close(); + } }); }); diff --git a/scripts/check-public-docs.ts b/scripts/check-public-docs.ts index 2e3479d5..ea8c58bf 100644 --- a/scripts/check-public-docs.ts +++ b/scripts/check-public-docs.ts @@ -14,9 +14,17 @@ const requiredPages = [ "configuration.mdx", "code-mode.mdx", "capabilities.mdx", + "catalog.mdx", + "dashboard.mdx", + "vault.mdx", + "project-binding.mdx", + "storage.mdx", + "privacy.mdx", + "privacy/indexing.mdx", "agent-integrations.mdx", "remote-attach.mdx", "troubleshooting.mdx", + "reference/cli.mdx", "reference/config.mdx", "reference/code-mode-api.mdx", "reference/caplet-files.mdx", @@ -48,8 +56,26 @@ const requiredContent = new Map([ ], ["capabilities.mdx", ["CAPLET.md", "OpenAPI", "GraphQL", "MCP"]], ["agent-integrations.mdx", ["Codex", "Claude", "OpenCode", "Pi"]], + ["catalog.mdx", ["entryKey", "catalog.caplets.dev/api/v1/catalog"]], + ["dashboard.mdx", ["Operator Client", "caplets remote host approve"]], + ["vault.mdx", ["AES-256-GCM", "CAPLETS_ENCRYPTION_KEY"]], + [ + "project-binding.mdx", + ["projectBinding", "--once", "does not start a sync session", ".capletsignore"], + ], + ["storage.mdx", ["caplets storage schema-migrate", "caplets storage migrate-legacy"]], + [ + "privacy.mdx", + [ + "CAPLETS_DISABLE_TELEMETRY", + "CAPLETS_DISABLE_UPDATE_CHECK", + "CAPLETS_DISABLE_CATALOG_INDEXING", + ], + ], + ["privacy/indexing.mdx", ["CAPLETS_DISABLE_CATALOG_INDEXING", "full installed `CAPLET.md`"]], ["remote-attach.mdx", ["caplets attach", "caplets remote login"]], ["troubleshooting.mdx", ["caplets doctor", "CAPLETS_CONFIG"]], + ["reference/cli.mdx", ["caplets completion", "caplets storage records", "CAPLETS_MODE"]], [ "reference/config.mdx", [ diff --git a/scripts/generate-docs-reference.ts b/scripts/generate-docs-reference.ts index ec47e0df..40727fbe 100644 --- a/scripts/generate-docs-reference.ts +++ b/scripts/generate-docs-reference.ts @@ -124,6 +124,7 @@ function schemaPage({ const majorSections = rows.filter(({ name }) => [ + "storage", "serve", "completion", "options", @@ -171,6 +172,9 @@ ${majorSections.map((row) => sectionDetails(row.name, schema.properties?.[row.na function sectionDetails(name: string, schema: JsonSchema | undefined, sourcePath: string): string { if (!schema) return `### \`${name}\`\n\nNo generated details are available.`; + if (sourcePath === "schemas/caplets-config.schema.json" && name === "storage") { + return storageConfigSection(schema); + } if (sourcePath === "schemas/caplet.schema.json" && name === "cliTools") { return cliToolsCapletFileSection(schema); } @@ -208,6 +212,40 @@ ${schema.description ?? "Nested fields from the canonical schema."} ${rows.join("\n")}`; } +function storageConfigSection(schema: JsonSchema): string { + return `### \`storage\` + +${schema.description ?? "Authoritative Host State backend configuration."} + +#### SQLite + +| Field | Status | Type | Description | +| --- | --- | --- | --- | +| \`type\` | Required | "sqlite" | Select SQLite Authoritative Host State. | +| \`path\` | Optional | string | SQLite database path. The platform state directory is used when omitted. | +| \`assets\` | Optional | object | SQL or S3-compatible Caplet bundle asset storage. | +| \`bundleLimits\` | Optional | object | Per-bundle auxiliary file count and byte limits. | + +#### PostgreSQL + +| Field | Status | Type | Description | +| --- | --- | --- | --- | +| \`type\` | Required | "postgres" | Select PostgreSQL Authoritative Host State. | +| \`connectionString\` | Required | string | PostgreSQL bootstrap connection URL. | +| \`schema\` | Optional | string | Logical-host schema; defaults to \`caplets\`. | +| \`assets\` | Optional | object | SQL or S3-compatible Caplet bundle asset storage. | +| \`bundleLimits\` | Optional | object | Per-bundle auxiliary file count and byte limits. | + +\`assets.type\` is \`sql\` or \`s3\`. S3 requires \`region\` and \`bucket\`; optional fields are +\`endpoint\`, \`prefix\`, \`forcePathStyle\`, \`accessKeyId\`, and \`secretAccessKey\`. Static access +keys must be configured together. Prefer workload identity or the AWS SDK credential chain. + +\`bundleLimits.maxFiles\` accepts integers from 1 through 100,000. \`maxFileBytes\` and +\`maxTotalBytes\` accept positive integers. +See [Authoritative Host State](/storage/) before PostgreSQL schema migration, S3 setup, backup, or +legacy migration.`; +} + function cliToolsCapletFileSection(schema: JsonSchema): string { return `### \`cliTools\` From b6dbac86d62c961eed512f3d1bdb7604184ac7a1 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Sun, 19 Jul 2026 05:26:10 -0400 Subject: [PATCH 2/2] test: cover shared legacy admin state --- packages/core/src/cli.ts | 8 +- packages/core/src/storage/legacy-migration.ts | 2 + .../core/test/storage-records-cli.test.ts | 77 ++++++++++++++++++- 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/packages/core/src/cli.ts b/packages/core/src/cli.ts index c4780d03..3c89a368 100644 --- a/packages/core/src/cli.ts +++ b/packages/core/src/cli.ts @@ -4177,7 +4177,9 @@ function configureStorageCommands( const legacyVaultRoot = context.io.authDir ? join(authDir, "vault") : join(defaultStateBaseDir(context.env), "caplets", "vault"); - const remoteSecurityDir = join(authDir, "remote-server"); + // Legacy remote credentials and dashboard activity intentionally share this directory; + // each store owns a distinct file within it. + const legacyHostAdminDir = join(authDir, "remote-server"); const report = await migrateLegacyHostState({ storage: loadHostStorageConfig(configPath), capletsRoot, @@ -4187,9 +4189,9 @@ function configureStorageCommands( legacyVaultRoot, legacyVaultEnv: context.env, targetVaultRoot: legacyVaultRoot, - remoteSecurityDir, + remoteSecurityDir: legacyHostAdminDir, setupStateDir: join(defaultCacheBaseDir(context.env), "caplets", "setup"), - operatorActivityDir: remoteSecurityDir, + operatorActivityDir: legacyHostAdminDir, ...(options.backupRoot ? { backupRoot: options.backupRoot } : {}), dryRun: options.dryRun ?? false, }); diff --git a/packages/core/src/storage/legacy-migration.ts b/packages/core/src/storage/legacy-migration.ts index b11bcba8..0ca612f3 100644 --- a/packages/core/src/storage/legacy-migration.ts +++ b/packages/core/src/storage/legacy-migration.ts @@ -528,6 +528,8 @@ async function vaultGrantsAvailableBeforeImport( storage: HostStorage, grants: LegacyVaultGrantImport[], ): Promise { + // Stored-record grants for records in this migration become importable only after those + // records are inserted in the transaction. const available: LegacyVaultGrantImport[] = []; for (const grant of grants) { if (grant.originKind !== "stored-record" || (await storage.caplets.get(grant.capletId))) { diff --git a/packages/core/test/storage-records-cli.test.ts b/packages/core/test/storage-records-cli.test.ts index a9972bb7..bcc32968 100644 --- a/packages/core/test/storage-records-cli.test.ts +++ b/packages/core/test/storage-records-cli.test.ts @@ -3,6 +3,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { writeTokenBundle } from "../src/auth/store"; +import { DashboardActivityLog } from "../src/dashboard/activity-log"; +import { RemoteServerCredentialStore } from "../src/remote/server-credential-store"; import { FileVaultStore } from "../src/vault"; import { runCli } from "../src/cli"; import { installCaplets } from "../src/install"; @@ -292,6 +294,41 @@ describe("stored Caplet Record CLI", () => { }, authDir, ); + const remoteDir = join(authDir, "remote-server"); + const hostUrl = "https://host.example.test/caplets"; + const remoteNow = new Date(); + const remote = new RemoteServerCredentialStore({ dir: remoteDir }); + const usedPairing = remote.createPairingCode({ + hostUrl, + clientLabel: "Migrated client", + ttlMs: 60 * 60_000, + now: remoteNow, + }); + const remoteCredentials = remote.exchangePairingCode({ + hostUrl, + code: usedPairing.code, + clientLabel: "Migrated client", + now: remoteNow, + }); + const unusedPairing = remote.createPairingCode({ + hostUrl, + clientLabel: "Pair after migration", + ttlMs: 60 * 60_000, + now: remoteNow, + }); + const pendingLogin = remote.createPendingLogin({ + hostUrl, + clientLabel: "Pending client", + sourceHint: "migration-test", + now: remoteNow, + }); + const activity = new DashboardActivityLog({ dir: remoteDir }).append({ + actorClientId: "legacy_operator", + action: "catalog_installed", + target: { type: "catalog", id: "legacy-cli", label: "Legacy CLI" }, + metadata: { source: "legacy" }, + now: new Date("2026-07-18T10:04:00.000Z"), + }); const vault = new FileVaultStore({ root: vaultRoot, env }); vault.set("LEGACY_TOKEN", "legacy-vault-value"); vault.grantAccess({ @@ -314,12 +351,12 @@ describe("stored Caplet Record CLI", () => { backendAuthTokenBundles: 1, vaultValues: 1, vaultGrants: 2, - remotePairingCodes: 0, - remoteClients: 0, - remotePendingLogins: 0, + remotePairingCodes: 2, + remoteClients: 1, + remotePendingLogins: 1, setupApprovals: 0, setupAttempts: 0, - operatorActivityEntries: 0, + operatorActivityEntries: 1, dashboardSessions: "not_applicable_no_legacy_format", projectBindings: "not_applicable_no_legacy_format", }; @@ -357,8 +394,16 @@ describe("stored Caplet Record CLI", () => { }); expect(existsSync(join(authDir, "legacy-cli.json"))).toBe(false); expect(existsSync(vault.valuePath("LEGACY_TOKEN"))).toBe(false); + expect(existsSync(join(remoteDir, "remote-server-credentials.json"))).toBe(false); + expect(existsSync(join(remoteDir, "dashboard-activity.jsonl"))).toBe(false); expect(existsSync(vault.paths.keyFile)).toBe(true); expect(existsSync(join(migrated.backupPath, "vault", "vault-key"))).toBe(true); + expect( + existsSync(join(migrated.backupPath, "remote-security", "remote-server-credentials.json")), + ).toBe(true); + expect( + existsSync(join(migrated.backupPath, "operator-activity", "dashboard-activity.jsonl")), + ).toBe(true); expect(existsSync(overlayPath)).toBe(true); const storage = await createHostStorage({ type: "sqlite", path: databasePath }, { vaultRoot }); @@ -385,6 +430,30 @@ describe("stored Caplet Record CLI", () => { }), ]), ); + await expect( + storage.remoteSecurity.validateAccessToken({ + hostUrl, + accessToken: remoteCredentials.accessToken, + }), + ).resolves.toMatchObject({ clientId: remoteCredentials.clientId, role: "access" }); + await expect( + storage.remoteSecurity.pollPendingLogin({ + flowId: pendingLogin.flowId, + pendingCompletionSecret: pendingLogin.pendingCompletionSecret, + }), + ).resolves.toEqual({ flowId: pendingLogin.flowId, status: "pending" }); + await expect( + storage.remoteSecurity.exchangePairingCode({ + hostUrl, + code: unusedPairing.code, + clientLabel: "Post-migration client", + }), + ).resolves.toMatchObject({ hostUrl, role: "access" }); + await expect(storage.operatorActivity.list({ action: "catalog_installed" })).resolves.toEqual( + { + entries: [expect.objectContaining({ id: activity.id, actorClientId: "legacy_operator" })], + }, + ); } finally { await storage.close(); }