From f996effdea1c42f3345072ae668214f6e2fa6b22 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Wed, 24 Jun 2026 08:26:25 -0400 Subject: [PATCH 1/6] feat: add anonymous telemetry --- .changeset/anonymous-telemetry.md | 8 + .github/workflows/release.yml | 8 + CONCEPTS.md | 6 + README.md | 17 + .../src/content/docs/agent-integrations.mdx | 6 + apps/docs/src/content/docs/install.mdx | 22 + .../src/content/docs/reference/config.mdx | 33 +- .../docs/src/content/docs/troubleshooting.mdx | 27 ++ apps/landing/public/config.schema.json | 4 + ...-06-24-anonymous-telemetry-requirements.md | 228 +++++++++ ...06-24-001-feat-anonymous-telemetry-plan.md | 298 ++++++++++++ docs/product/anonymous-telemetry.md | 17 + docs/product/telemetry-provider-readiness.md | 46 ++ docs/product/telemetry-readout.md | 25 + package.json | 1 + packages/core/package.json | 2 + packages/core/src/cli.ts | 439 ++++++++++++++++- packages/core/src/cli/code-mode.ts | 15 +- packages/core/src/cli/commands.ts | 3 + packages/core/src/config.ts | 22 +- packages/core/src/config/paths.ts | 44 ++ packages/core/src/engine.ts | 146 +++++- packages/core/src/native/service.ts | 52 +- packages/core/src/serve/session.ts | 9 + packages/core/src/telemetry/context.ts | 80 ++++ packages/core/src/telemetry/debug.ts | 18 + packages/core/src/telemetry/delivery.ts | 5 + packages/core/src/telemetry/events.ts | 175 +++++++ packages/core/src/telemetry/identity.ts | 6 + packages/core/src/telemetry/index.ts | 64 +++ packages/core/src/telemetry/notice.ts | 21 + packages/core/src/telemetry/privacy.ts | 152 ++++++ packages/core/src/telemetry/providers.ts | 141 ++++++ packages/core/src/telemetry/runtime.ts | 205 ++++++++ packages/core/src/telemetry/state.ts | 181 +++++++ packages/core/test/config.test.ts | 64 +++ packages/core/test/native.test.ts | 72 ++- packages/core/test/telemetry-cli.test.ts | 199 ++++++++ packages/core/test/telemetry-docs.test.ts | 53 +++ packages/core/test/telemetry-events.test.ts | 126 +++++ .../core/test/telemetry-providers.test.ts | 190 ++++++++ .../core/test/telemetry-redaction.test.ts | 46 ++ packages/core/test/telemetry-release.test.ts | 46 ++ packages/core/test/telemetry-state.test.ts | 181 +++++++ packages/opencode/README.md | 8 + packages/opencode/src/index.ts | 7 +- packages/opencode/test/opencode.test.ts | 2 + packages/pi/README.md | 8 + packages/pi/src/index.ts | 6 +- packages/pi/test/pi.test.ts | 24 +- pnpm-lock.yaml | 448 +++++++++++++++++- schemas/caplets-config.schema.json | 4 + scripts/check-telemetry-release-env.ts | 79 +++ 53 files changed, 4033 insertions(+), 56 deletions(-) create mode 100644 .changeset/anonymous-telemetry.md create mode 100644 docs/brainstorms/2026-06-24-anonymous-telemetry-requirements.md create mode 100644 docs/plans/2026-06-24-001-feat-anonymous-telemetry-plan.md create mode 100644 docs/product/anonymous-telemetry.md create mode 100644 docs/product/telemetry-provider-readiness.md create mode 100644 docs/product/telemetry-readout.md create mode 100644 packages/core/src/telemetry/context.ts create mode 100644 packages/core/src/telemetry/debug.ts create mode 100644 packages/core/src/telemetry/delivery.ts create mode 100644 packages/core/src/telemetry/events.ts create mode 100644 packages/core/src/telemetry/identity.ts create mode 100644 packages/core/src/telemetry/index.ts create mode 100644 packages/core/src/telemetry/notice.ts create mode 100644 packages/core/src/telemetry/privacy.ts create mode 100644 packages/core/src/telemetry/providers.ts create mode 100644 packages/core/src/telemetry/runtime.ts create mode 100644 packages/core/src/telemetry/state.ts create mode 100644 packages/core/test/telemetry-cli.test.ts create mode 100644 packages/core/test/telemetry-docs.test.ts create mode 100644 packages/core/test/telemetry-events.test.ts create mode 100644 packages/core/test/telemetry-providers.test.ts create mode 100644 packages/core/test/telemetry-redaction.test.ts create mode 100644 packages/core/test/telemetry-release.test.ts create mode 100644 packages/core/test/telemetry-state.test.ts create mode 100644 scripts/check-telemetry-release-env.ts diff --git a/.changeset/anonymous-telemetry.md b/.changeset/anonymous-telemetry.md new file mode 100644 index 00000000..56eba6c5 --- /dev/null +++ b/.changeset/anonymous-telemetry.md @@ -0,0 +1,8 @@ +--- +"@caplets/core": minor +"caplets": minor +"@caplets/opencode": minor +"@caplets/pi": minor +--- + +Add opt-out anonymous telemetry controls, privacy-gated event builders, Sentry/PostHog provider adapters, CLI status/debug commands, and stderr-only first-run disclosure for eligible CLI/runtime commands. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cdfa2416..b396c41a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,6 +51,12 @@ jobs: - name: Verify package run: pnpm verify + - name: Check telemetry release env + env: + CAPLETS_POSTHOG_TOKEN: ${{ secrets.CAPLETS_POSTHOG_TOKEN }} + CAPLETS_SENTRY_DSN: ${{ secrets.CAPLETS_SENTRY_DSN }} + run: pnpm telemetry:check-release-env + - name: Create release PR or publish id: changesets uses: changesets/action@v1 @@ -60,6 +66,8 @@ jobs: commit: "chore: version packages" title: "chore: version packages" env: + CAPLETS_POSTHOG_TOKEN: ${{ secrets.CAPLETS_POSTHOG_TOKEN }} + CAPLETS_SENTRY_DSN: ${{ secrets.CAPLETS_SENTRY_DSN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Check whether CLI package was published diff --git a/CONCEPTS.md b/CONCEPTS.md index 1a43fbf5..327ba809 100644 --- a/CONCEPTS.md +++ b/CONCEPTS.md @@ -78,6 +78,12 @@ Recovery References are separate from session handles. Possessing a session hand The Caplets exposure mode where agents discover and call backend operations through a small set of wrapper tools instead of receiving every downstream operation as a separate top-level tool. +### Anonymous Telemetry + +Opt-out Caplets usage and reliability reporting that uses a stable anonymous installation ID and categorical metadata only. + +Anonymous Telemetry is split by purpose: PostHog receives product usage events, while Sentry receives sanitized reliability events. It must not collect raw config, prompts, Code Mode code, tool arguments, tool outputs, paths, URLs, hostnames, Caplet IDs, credentials, or unsanitized error payloads. + ## Remote Attach ### Remote Attach diff --git a/README.md b/README.md index 17b166ca..44797ce2 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,23 @@ caplets vault access grant GH_TOKEN github --remote Vault values are not exposed through Code Mode, progressive tools, or native agent APIs. Unset or ungranted Vault references quarantine only the affected Caplet and appear in `caplets doctor`. +## Anonymous Telemetry + +Caplets collects opt-out anonymous telemetry for product usage and reliability. The first eligible +interactive CLI run writes a notice to stderr only, including both disable controls: + +```sh +CAPLETS_DISABLE_TELEMETRY=1 caplets serve +caplets telemetry disable +``` + +Use `caplets telemetry status`, `enable`, `disable`, `rotate-id`, `delete-id`, and `debug` to +inspect or control local telemetry. Caplets never collects raw config, prompts, Code Mode code, tool +arguments, tool outputs, logs, file paths, URLs, hostnames, Caplet IDs, credentials, tokens, raw +environment variables, raw error messages, or unsanitized stack traces. Rotating or deleting the +local anonymous ID does not delete provider-side historical anonymous events; provider retention +controls historical data. + ## Benchmark The deterministic benchmark compares flat MCP exposure with Caplets over the same mock diff --git a/apps/docs/src/content/docs/agent-integrations.mdx b/apps/docs/src/content/docs/agent-integrations.mdx index c081ae85..f79e9aa7 100644 --- a/apps/docs/src/content/docs/agent-integrations.mdx +++ b/apps/docs/src/content/docs/agent-integrations.mdx @@ -41,6 +41,9 @@ claude mcp add --transport stdio --scope user caplets -- caplets serve OpenCode can use Caplets through MCP or the native OpenCode integration. Native mode exposes `caplets__code_mode` for Code Mode, plus progressive or direct tools only when configured. +Native integrations use the shared anonymous telemetry controls and do not send telemetry on a +native-first run until a visible CLI telemetry notice has already been recorded. + Remote native usage: ```sh @@ -52,6 +55,9 @@ CAPLETS_MODE=remote CAPLETS_REMOTE_URL=https://caplets.example.com/caplets openc Pi can use Caplets through MCP or the native Pi integration. Native mode uses the same local and remote selection rules as OpenCode. +Native integrations use the shared anonymous telemetry controls and do not send telemetry on a +native-first run until a visible CLI telemetry notice has already been recorded. + Remote native usage: ```sh diff --git a/apps/docs/src/content/docs/install.mdx b/apps/docs/src/content/docs/install.mdx index 8c91d0b6..4c7240a7 100644 --- a/apps/docs/src/content/docs/install.mdx +++ b/apps/docs/src/content/docs/install.mdx @@ -77,6 +77,28 @@ npx caplets doctor The output should show the active Caplets paths and any integration checks Caplets can run in this environment. Treat failed rows as setup work, not as agent bugs. +## Anonymous telemetry + +Caplets collects opt-out anonymous telemetry for product usage and reliability. The first eligible +interactive CLI run writes a notice to stderr only, so JSON stdout and MCP protocol output stay +clean. + +Disable telemetry for one process: + +```sh +CAPLETS_DISABLE_TELEMETRY=1 caplets serve +``` + +Disable it in your user Caplets config: + +```sh +caplets telemetry disable +``` + +Inspect local state with `caplets telemetry status`. Caplets never collects raw config, prompts, +Code Mode code, tool arguments, tool outputs, logs, file paths, URLs, hostnames, Caplet IDs, +credentials, tokens, raw environment variables, raw error messages, or unsanitized stack traces. + ## First agent check After `doctor` is clean enough for your integration, ask the agent: diff --git a/apps/docs/src/content/docs/reference/config.mdx b/apps/docs/src/content/docs/reference/config.mdx index caf5c7ba..82970532 100644 --- a/apps/docs/src/content/docs/reference/config.mdx +++ b/apps/docs/src/content/docs/reference/config.mdx @@ -62,22 +62,23 @@ Public OpenAPI endpoint: ## Top-level fields -| Field | Status | Type | Description | -| --------------------- | -------- | ------- | ------------------------------------------------------------- | -| `$schema` | Optional | string | Optional JSON Schema for editor validation. | -| `version` | Optional | number | Caplets config schema version. | -| `defaultSearchLimit` | Optional | integer | Default maximum number of same-server search results. | -| `maxSearchLimit` | Optional | integer | Maximum accepted search_tools limit. | -| `completion` | Optional | object | Shell completion discovery timeout and cache settings. | -| `options` | Optional | object | Global Caplets runtime options. | -| `namespaceAliases` | Optional | object | Source-level namespace aliases for hash-qualified Caplet IDs. | -| `mcpServers` | Optional | object | Downstream MCP servers keyed by stable server ID. | -| `openapiEndpoints` | Optional | object | OpenAPI endpoints keyed by stable Caplet ID. | -| `googleDiscoveryApis` | Optional | object | Google Discovery APIs keyed by stable Caplet ID. | -| `graphqlEndpoints` | Optional | object | GraphQL endpoints keyed by stable Caplet ID. | -| `httpApis` | Optional | object | HTTP APIs keyed by stable Caplet ID. | -| `cliTools` | Optional | object | CLI tools keyed by stable Caplet ID. | -| `capletSets` | Optional | object | Nested Caplet collections keyed by stable Caplet ID. | +| Field | Status | Type | Description | +| --------------------- | -------- | ------- | ---------------------------------------------------------------------- | +| `$schema` | Optional | string | Optional JSON Schema for editor validation. | +| `version` | Optional | number | Caplets config schema version. | +| `defaultSearchLimit` | Optional | integer | Default maximum number of same-server search results. | +| `maxSearchLimit` | Optional | integer | Maximum accepted search_tools limit. | +| `telemetry` | Optional | boolean | Set false to disable anonymous Caplets telemetry for this user config. | +| `completion` | Optional | object | Shell completion discovery timeout and cache settings. | +| `options` | Optional | object | Global Caplets runtime options. | +| `namespaceAliases` | Optional | object | Source-level namespace aliases for hash-qualified Caplet IDs. | +| `mcpServers` | Optional | object | Downstream MCP servers keyed by stable server ID. | +| `openapiEndpoints` | Optional | object | OpenAPI endpoints keyed by stable Caplet ID. | +| `googleDiscoveryApis` | Optional | object | Google Discovery APIs keyed by stable Caplet ID. | +| `graphqlEndpoints` | Optional | object | GraphQL endpoints keyed by stable Caplet ID. | +| `httpApis` | Optional | object | HTTP APIs keyed by stable Caplet ID. | +| `cliTools` | Optional | object | CLI tools keyed by stable Caplet ID. | +| `capletSets` | Optional | object | Nested Caplet collections keyed by stable Caplet ID. | ## Major sections diff --git a/apps/docs/src/content/docs/troubleshooting.mdx b/apps/docs/src/content/docs/troubleshooting.mdx index a200a3a2..26e46828 100644 --- a/apps/docs/src/content/docs/troubleshooting.mdx +++ b/apps/docs/src/content/docs/troubleshooting.mdx @@ -64,6 +64,33 @@ CAPLETS_CONFIG=/path/to/caplets.json caplets doctor Project config should live at `.caplets/config.json`. User config is for personal agent setup; project config is for capabilities that should travel with a repository. +### Disable or inspect telemetry + +Expected symptom: you want to disable anonymous telemetry, inspect local telemetry state, or see the +sanitized events Caplets would send. + +Disable telemetry for one process: + +```sh +CAPLETS_DISABLE_TELEMETRY=1 caplets doctor +``` + +Disable telemetry in your user Caplets config: + +```sh +caplets telemetry disable +``` + +Inspect state or run a local debug capture: + +```sh +caplets telemetry status +caplets telemetry debug -- doctor +``` + +`caplets telemetry rotate-id` and `caplets telemetry delete-id` affect only the local anonymous ID. +They do not delete provider-side historical anonymous events. + ### Code Mode returns `ok: false` Expected symptom: a Code Mode call returns an error envelope instead of tool data. diff --git a/apps/landing/public/config.schema.json b/apps/landing/public/config.schema.json index 90021fe3..46cbc33b 100644 --- a/apps/landing/public/config.schema.json +++ b/apps/landing/public/config.schema.json @@ -29,6 +29,10 @@ "exclusiveMinimum": 0, "maximum": 50 }, + "telemetry": { + "description": "Set false to disable anonymous Caplets telemetry for this user config.", + "type": "boolean" + }, "completion": { "default": { "discoveryTimeoutMs": 750, diff --git a/docs/brainstorms/2026-06-24-anonymous-telemetry-requirements.md b/docs/brainstorms/2026-06-24-anonymous-telemetry-requirements.md new file mode 100644 index 00000000..cdd8dcfc --- /dev/null +++ b/docs/brainstorms/2026-06-24-anonymous-telemetry-requirements.md @@ -0,0 +1,228 @@ +--- +date: 2026-06-24 +topic: anonymous-telemetry +--- + +# Anonymous Telemetry Requirements + +## Summary + +Add opt-out anonymous telemetry across the Caplets CLI, native integrations, and remote or Cloud attach flows. Sentry captures sanitized reliability data, while PostHog captures feature-level product usage with categorical metadata only. + +--- + +## Problem Frame + +Caplets needs broader usage data to decide where to invest. The current product questions span setup drop-off, CLI versus native adoption, local versus remote or Cloud usage, Code Mode versus progressive or direct exposure, and backend-family investment. + +Crash reporting alone does not answer those questions. Coarse lifecycle telemetry also falls short because it shows that Caplets ran without explaining which product surfaces created value. + +--- + +## Product Rationale + +Opt-out telemetry is a deliberate product tradeoff. Caplets needs enough aggregate data to understand setup friction, surface adoption, runtime mode, exposure mode, backend-family usage, and reliability without requiring every user to notice and approve a prompt first. + +The trust risk is real because Caplets runs near config, prompts, tools, local files, and agent workflows. The implementation must make the data boundaries concrete, show a notice before any user-visible first event leaves the process, provide obvious disable controls, and revisit the opt-out default if telemetry complaints, disable rates, or support feedback show that the default is harming adoption. + +--- + +## Decision Questions + +The first implementation should answer these product questions with the minimum viable event set and a lightweight readout path such as saved provider queries or a recurring usage report. + +- **Setup funnel:** Where do users drop between init, setup, add, auth, remote login, doctor, first serve or attach, and first successful tool execution? +- **Surface adoption:** Which surfaces produce real usage: CLI, OpenCode, Pi, MCP serving, native service, remote attach, and Cloud attach? +- **Runtime investment:** Are users primarily local, self-hosted remote, or hosted Cloud, and how much of that use is CI or non-interactive automation? +- **Exposure-mode investment:** Are users succeeding with Code Mode, progressive tools, direct tools, or mixed exposure modes? +- **Backend-family investment:** Which backend families are configured and actually invoked enough to justify deeper investment? +- **Reliability investment:** Which sanitized error codes, command families, native integrations, and runtime categories create the most user-visible failure pressure? + +These questions should be reviewed on a recurring cadence. Telemetry should drive investment only when event volume and delivery health are good enough to distinguish real non-usage from missing or failed telemetry. + +--- + +## Key Decisions + +- **Use feature-level anonymous telemetry.** Caplets should collect enough categorical detail to connect setup, runtime mode, exposure mode, backend family, native integration, and outcome. +- **Split responsibilities between providers.** PostHog is for product usage events; Sentry is for sanitized errors and reliability. +- **Default to opt-out.** Anonymous telemetry is enabled by default, but users can disable it durably or per process. +- **Use a stable anonymous installation ID.** The ID is needed to connect setup, later usage, and retention without identifying a person. +- **Collect CI usage, suppress tests.** CI and non-interactive runs are real automation usage; test environments are noise and should not send telemetry. +- **Prefer ongoing transparency over repeated notices.** Caplets should show a first-run notice on stderr and expose durable status, debug, enable, and disable commands. + +--- + +## Actors + +- A1. **Caplets user:** Runs the CLI, installs Caplets, configures Caplets, and expects clear privacy controls. +- A2. **Coding agent:** Uses Caplets through CLI-backed workflows, MCP serving, Code Mode, progressive tools, or direct tools. +- A3. **Native integration user:** Uses Caplets through OpenCode or Pi native integrations. +- A4. **Caplets maintainer:** Reviews aggregate usage and reliability data to prioritize product work. +- A5. **Telemetry providers:** Sentry and PostHog receive only the approved anonymous payloads. + +--- + +## Requirements + +**Consent and controls** + +- R1. Caplets must enable anonymous telemetry by default unless a local or process-level control disables it. +- R2. `telemetry: false` in Caplets `config.json` must disable both PostHog and Sentry telemetry. +- R3. `CAPLETS_DISABLE_TELEMETRY=1` must disable both PostHog and Sentry telemetry for the current process. +- R4. `CAPLETS_DISABLE_TELEMETRY=1` must take precedence over config and command-based enablement. +- R5. Caplets must add `caplets telemetry status`, `caplets telemetry enable`, `caplets telemetry disable`, and `caplets telemetry debug`. +- R6. `caplets telemetry status` must explain whether telemetry is enabled, which control decided that state, and whether an anonymous installation ID exists. +- R7. `caplets telemetry debug` must show the sanitized events Caplets would send without sending them. + +**Notice and transparency** + +- R8. Caplets must show a first-run telemetry notice on stderr before sending the first telemetry event. +- R9. The first-run notice must include the fact that Caplets collects anonymous telemetry and the exact disable controls. +- R10. The first-run notice must never write to stdout. +- R11. The first-run notice gate must cover every telemetry-emitting CLI, daemon, attach, serve, setup, auth, doctor, native integration, and tool-execution surface before that surface sends its first telemetry event. +- R12. Native-first telemetry must either use a user-visible notice or status channel before telemetry is sent, or suppress native telemetry until a durable CLI or status notice has been recorded. +- R13. CI, daemon, native, and other non-interactive or hidden-stderr runs must not mark the telemetry notice as shown unless the notice was actually user-visible. +- R14. Repeated `serve`, `attach`, daemon, and native integration runs must not print repeated telemetry notices after a user-visible telemetry notice has been recorded. + +**Identity and environment** + +- R15. Caplets must use one stable anonymous installation ID for telemetry correlation. +- R16. The anonymous installation ID must not be derived from names, emails, paths, hostnames, URLs, config contents, remote credentials, or Caplet IDs. +- R17. Caplets must define anonymous installation ID generation, storage scope, reset behavior, and behavior after telemetry disablement and re-enablement. +- R18. `caplets telemetry status` must show whether an anonymous installation ID exists, and telemetry controls must provide a way to delete or rotate the local anonymous installation ID. +- R19. CI and non-interactive telemetry events must be collected by default and tagged with categorical CI or non-interactive state. +- R20. CI and ephemeral automation must have explicit identity handling so setup-to-retention analysis does not merge unrelated environments or fragment one durable installation unexpectedly. +- R21. Test environments must suppress telemetry automatically. + +**PostHog product events** + +- R22. PostHog events must cover setup and activation milestones including init, setup, add, auth, remote login, doctor, serve, attach, daemon, native startup, and first successful tool execution. +- R23. PostHog events must distinguish surface category: CLI, OpenCode, Pi, MCP serving, native service, remote attach, and Cloud attach. +- R24. PostHog events must distinguish runtime mode category: local, self-hosted remote, and hosted Cloud. +- R25. PostHog events must distinguish execution-context category such as interactive, non-interactive, and CI separately from runtime mode. +- R26. PostHog events must include backend-family counts using only categories such as MCP, OpenAPI, Google Discovery API, GraphQL, HTTP, CLI tools, and Caplet sets. +- R27. PostHog events must include exposure-mode counts using only categories such as Code Mode, progressive, direct, and mixed exposure modes. +- R28. PostHog events must capture Code Mode outcome categories including success, failure, timeout bucket, duration bucket, session created or reused, diagnostic codes, and whether any Caplet was invoked. +- R29. PostHog events must capture operation-family categories for progressive and CLI operations, such as inspect, check, tools, search, describe, call, resources, prompts, and complete. +- R30. PostHog event transport must not delay or fail the primary CLI, server, attach, daemon, or native integration workflow. + +**Sentry reliability events** + +- R31. Sentry must capture sanitized Caplets errors from CLI and native integration surfaces. +- R32. Sentry events must include categorical context such as package name, package version, surface, runtime mode, command family, Caplets error code, operating system family, architecture, and Node major version. +- R33. Sentry must not receive raw config, prompts, Code Mode code, tool arguments, tool outputs, resource contents, prompt contents, file paths, URLs, hostnames, Caplet IDs, environment variables, credentials, tokens, or unsanitized stack traces. +- R34. Sentry and PostHog disablement must be evaluated before provider clients are initialized. + +**Privacy boundaries** + +- R35. Telemetry must never collect raw config, prompts, Code Mode code, tool arguments, tool outputs, logs, resource contents, prompt contents, file paths, URLs, hostnames, Caplet IDs, credential values, token values, or raw environment variables. +- R36. Telemetry must only use allowlisted event names and allowlisted property keys. +- R37. Telemetry properties must be categorical, boolean, numeric count, or bucketed timing values unless a requirement explicitly allows another shape. +- R38. Provider-side IP capture and geolocation enrichment must be disabled or scrubbed for Caplets telemetry projects where the provider supports it. +- R39. Provider-side event retention limits must be defined for both PostHog product events and Sentry reliability events. +- R40. Local telemetry redaction must happen before data leaves the process. + +**Telemetry quality and provider operations** + +- R41. Distributed Sentry and PostHog identifiers must be intake-only, environment-specific, revocable, and must not include management API keys, private tokens, or credentials with read or admin access to telemetry data. +- R42. Telemetry transport must remain fire-and-forget for users, but provider initialization failures, event drops, delivery-health uncertainty, and provider ingestion issues must be observable in aggregate or explicitly accounted for in analysis. +- R43. Telemetry transport and provider projects must include abuse controls such as bounded queues, sampling or rate limiting for repeated failures and CI loops, and provider-side quota or ingestion monitoring. +- R44. The first implementation must include a lightweight analysis surface, such as saved provider queries or a recurring product usage report, that maps collected events back to the decision questions in this document. + +--- + +## Key Flows + +- F1. **First eligible CLI run** + - **Trigger:** A user runs an eligible command before any telemetry notice has been recorded. + - **Actors:** A1, A4, A5 + - **Steps:** Caplets resolves telemetry state, prints the notice to stderr if telemetry is enabled and stderr is user-visible, records that the notice was shown, and then emits the allowed event. + - **Outcome:** The user receives disable instructions before the first event leaves the process. + +- F2. **Telemetry-disabled run** + - **Trigger:** A user sets `telemetry: false` or `CAPLETS_DISABLE_TELEMETRY=1`. + - **Actors:** A1, A2, A3 + - **Steps:** Caplets resolves telemetry state before provider initialization and skips both PostHog and Sentry. + - **Outcome:** The primary workflow runs with no telemetry network traffic. + +- F3. **Native integration usage** + - **Trigger:** OpenCode or Pi initializes and executes Caplets native tools. + - **Actors:** A2, A3, A4, A5 + - **Steps:** Caplets verifies that telemetry is enabled and a user-visible notice has already been recorded, or uses a native-visible notice or status channel before recording native startup, service reload, tool execution outcome, and Code Mode outcome categories. + - **Outcome:** Maintainers can compare native integration usage against CLI and attach usage without collecting user content. + +- F4. **Sanitized error capture** + - **Trigger:** A CLI, attach, daemon, or native integration surface encounters a reportable error. + - **Actors:** A1, A3, A4, A5 + - **Steps:** Caplets converts the failure into a sanitized categorical error payload and sends it to Sentry only if telemetry is enabled. + - **Outcome:** Maintainers see reliability patterns without receiving local paths, raw stack traces, URLs, credentials, or user-provided content. + +--- + +## Acceptance Examples + +- AE1. **Covers R2, R34.** Given `telemetry: false` and no disabling environment variable, when a user runs `caplets doctor`, then neither Sentry nor PostHog is initialized or called. +- AE2. **Covers R3, R4, R34.** Given telemetry is enabled in config and `CAPLETS_DISABLE_TELEMETRY=1` is set, when a native integration starts, then neither Sentry nor PostHog is initialized or called. +- AE3. **Covers R8, R9, R10, R11.** Given telemetry is enabled and no notice has been recorded, when a user runs `caplets serve`, then Caplets writes the telemetry notice to stderr before sending the first event and writes no notice to stdout. +- AE4. **Covers R14.** Given the telemetry notice has already been recorded, when the user later starts `caplets attach`, then Caplets does not print the notice again. +- AE5. **Covers R7.** Given a user runs `caplets telemetry debug`, when Caplets would normally send a usage event, then the sanitized payload is printed locally and no provider request is sent. +- AE6. **Covers R19, R25.** Given telemetry is enabled in CI, when a setup command runs locally, then the PostHog event is sent with runtime mode `local` and execution context `CI`. +- AE7. **Covers R21.** Given a test environment is detected, when any Caplets command or native integration runs, then telemetry is suppressed automatically. +- AE8. **Covers R15, R22, R23, R24, R25.** Given a user completes remote login and later uses Cloud attach through a native integration, when telemetry is enabled, then PostHog can connect those categorical milestones through the stable anonymous installation ID. +- AE9. **Covers R26, R27, R35.** Given a config contains MCP and HTTP Caplets with mixed exposure modes, when Caplets emits a product event, then the event contains only backend-family and exposure-mode counts rather than Caplet IDs or config values. +- AE10. **Covers R28, R33, R35.** Given a Code Mode run fails diagnostics on user code, when telemetry is enabled, then telemetry may include diagnostic codes and duration bucket but must not include the code or logs. +- AE11. **Covers R31, R32, R33.** Given a command fails with a Caplets error, when Sentry capture is enabled, then Sentry receives the Caplets error code and categorical context but not file paths, URLs, raw stack traces, or raw error payloads. +- AE12. **Covers R38.** Given Caplets telemetry projects are configured in Sentry and PostHog, when provider-side privacy settings are inspected, then IP capture and geolocation enrichment are disabled or scrubbed where supported. +- AE13. **Covers R12, R13.** Given a user's first telemetry-capable action happens through a native integration or hidden-stderr daemon, when no user-visible notice channel is available, then Caplets suppresses telemetry instead of recording the notice as shown. +- AE14. **Covers R17, R18, R39.** Given a user disables telemetry, when they inspect telemetry status, then Caplets explains whether the anonymous installation ID still exists and offers the documented delete or rotate control. +- AE15. **Covers R41.** Given shipped package contents are inspected, when provider identifiers are present, then they are intake-only, environment-specific, revocable, and not read or admin credentials. +- AE16. **Covers R42.** Given provider delivery fails, when a Caplets command runs, then the command succeeds without telemetry blocking and aggregate delivery health records the failure or uncertainty. +- AE17. **Covers R43.** Given a CI loop repeatedly emits the same telemetry shape, when telemetry is enabled, then bounded queues, sampling, rate limiting, or provider quotas prevent unbounded ingestion. +- AE18. **Covers R44.** Given the first telemetry release is reviewed, when maintainers inspect product usage, then saved queries or a recurring report maps collected events back to the decision questions. + +--- + +## Success Criteria + +- Maintainers can answer which setup steps, integration surfaces, runtime modes, exposure modes, and backend families are actually used. +- Maintainers can map those answers to documented product decision questions, thresholds, and review cadence. +- Maintainers can compare Code Mode, progressive, and direct usage without inspecting user content. +- Maintainers can see sanitized error frequency by package, surface, runtime mode, and Caplets error code. +- Maintainers can detect when telemetry delivery health is too weak to distinguish missing events from real non-usage. +- Users can discover telemetry status and disable telemetry without reading source code. +- Telemetry cannot break MCP stdio, long-running serve or attach sessions, daemon workflows, or native integration startup. + +--- + +## Scope Boundaries + +- Capturing raw config, prompts, Code Mode code, tool arguments, tool outputs, logs, file paths, URLs, hostnames, Caplet IDs, and raw stack traces is out of scope. +- Full product analytics dashboards are out of scope for the first implementation, but saved queries or a recurring lightweight usage report are in scope. +- Per-user accounts, email identity, workspace identity, and organization-level identity are out of scope. +- Provider-side setup automation is out of scope; provider privacy configuration is a launch prerequisite. +- Self-hosted telemetry backends are out of scope for the first version. + +--- + +## Dependencies and Assumptions + +- The telemetry layer depends on Sentry and PostHog projects that are configured to minimize IP and geolocation capture. +- The first implementation assumes provider identifiers can be distributed as intake-only, environment-specific, revocable identifiers without exposing management, read, or admin access. +- The first implementation assumes event transport can be fire-and-forget for user workflows while retaining aggregate delivery-health visibility for analysis. +- The first implementation assumes docs can describe exactly what is collected, what is never collected, and how to disable or debug telemetry. + +--- + +## Sources / Research + +- `STRATEGY.md` frames runtime diagnosability, native agent surfaces, remote runtime, and public proof as active product priorities. +- `packages/cli/src/index.ts` is the published CLI wrapper that delegates to core CLI execution. +- `packages/core/src/cli.ts` centralizes CLI command handling, injected IO, env handling, remote routing, setup, daemon, attach, Code Mode, auth, vault, and operation commands. +- `packages/core/src/cli/commands.ts` defines top-level and nested CLI command families. +- `packages/core/src/native/service.ts` centralizes native service creation, local and remote mode resolution, tool listing, reload, and execution. +- `packages/opencode/src/index.ts` and `packages/opencode/src/hooks.ts` initialize the native service and execute native Caplets tools for OpenCode. +- `packages/pi/src/index.ts` initializes the native service, tracks Pi session lifecycle, and executes native Caplets tools for Pi. +- `packages/core/src/errors.ts` and `packages/core/src/redaction.ts` provide existing Caplets error codes and redaction helpers that telemetry should build on. +- Sentry server-side scrubbing docs: https://docs.sentry.io/security-legal-pii/scrubbing/server-side-scrubbing/ +- PostHog privacy controls docs: https://posthog.com/docs/product-analytics/privacy diff --git a/docs/plans/2026-06-24-001-feat-anonymous-telemetry-plan.md b/docs/plans/2026-06-24-001-feat-anonymous-telemetry-plan.md new file mode 100644 index 00000000..3827b9df --- /dev/null +++ b/docs/plans/2026-06-24-001-feat-anonymous-telemetry-plan.md @@ -0,0 +1,298 @@ +--- +title: "feat: Add anonymous telemetry" +type: feat +date: 2026-06-24 +origin: docs/brainstorms/2026-06-24-anonymous-telemetry-requirements.md +deepened: 2026-06-24 +--- + +# feat: Add anonymous telemetry + +## Summary + +Add opt-out anonymous telemetry across the Caplets CLI, MCP serve/attach flows, daemon workflows, Code Mode, and native integrations. PostHog captures categorical product usage events, while Sentry captures sanitized reliability events; both providers are gated by shared config/env controls, first-run transparency, and strict allowlisted payloads. + +## Problem Frame + +Caplets needs better aggregate product evidence for setup friction, CLI versus native usage, local versus remote or Cloud runtime adoption, Code Mode versus progressive/direct exposure, backend-family investment, and reliability pressure. The same surfaces run near local config, prompts, tool calls, Code Mode source, credentials, and filesystem paths, so telemetry must be designed as privacy-sensitive infrastructure rather than scattered provider calls. + +The implementation should make anonymous telemetry visible, controllable, and auditable without blocking the primary CLI, MCP, daemon, attach, or native-integration workflow. + +## Requirements + +**Consent, controls, and notice** + +- R1. Anonymous telemetry is enabled by default unless disabled by config, environment, test detection, or notice gating. +- R2. Top-level `telemetry: false` in user Caplets `config.json` disables both PostHog and Sentry. +- R3. `CAPLETS_DISABLE_TELEMETRY=1` disables both providers for the current process and takes precedence over config. +- R4. `caplets telemetry status`, `enable`, `disable`, `delete-id`, `rotate-id`, and `debug` expose local state and controls. +- R5. User-visible first-run notice writes only to stderr, includes the exact disable controls, and is recorded only when stderr is an interactive TTY or a tested integration-visible notice channel is used. +- R6. Native-first and hidden-stderr daemon surfaces suppress telemetry until a durable visible notice has already been recorded. +- R7. CI and non-interactive usage may send categorical telemetry without recording a visible-notice state; test environments suppress telemetry automatically. + +**Identity and event model** + +- R8. Telemetry uses a random stable anonymous installation ID stored in Caplets-owned state, never derived from names, paths, hosts, URLs, config, credentials, or Caplet IDs. +- R9. Product events cover setup and activation milestones, surface category, runtime mode, execution context, backend-family counts, exposure-mode counts, operation family, first successful tool execution, and Code Mode outcome categories. +- R10. Reliability events include only package/version, surface, runtime mode, command family, Caplets error code, operating system family, architecture, Node major version, and other approved categorical context. +- R11. Event names and property keys are allowlisted; properties are categorical, boolean, count, or bucketed timing values unless explicitly approved. + +**Privacy and provider operations** + +- R12. Telemetry never collects raw config, prompts, Code Mode code, tool arguments, tool outputs, logs, resource contents, prompt contents, file paths, URLs, hostnames, Caplet IDs, credentials, tokens, raw env, or unsanitized stack traces. +- R13. Disablement and test suppression are resolved before provider clients are initialized. +- R14. PostHog and Sentry identifiers shipped with packages are intake-only, environment-specific, revocable, and not management/read/admin credentials. +- R15. Provider-side IP capture, geolocation enrichment, scrubbing, retention, quota, and ingestion monitoring are launch prerequisites. +- R16. Telemetry is fire-and-forget for users, but provider initialization failures, event drops, and delivery-health uncertainty are observable for maintainers. +- R17. The first release includes a lightweight analysis surface mapping events back to the brainstorm's decision questions. + +## Key Technical Decisions + +- **KTD1. Centralize telemetry in `@caplets/core`.** CLI, serve, attach, daemon, Code Mode, and native packages already route through core boundaries, so the privacy gate, state resolver, event schemas, debug sink, and providers should live under `packages/core/src/telemetry/`. +- **KTD2. Use top-level user-config opt-out.** The user-facing key is `telemetry`, not an `options.telemetry` nested value. `caplets telemetry enable|disable` mutates the user config only; project config does not get a telemetry control in this release. +- **KTD3. Resolve state before importing or constructing provider clients.** The telemetry layer must short-circuit on env disablement, config disablement, test suppression, or missing visible notice before Sentry/PostHog clients are initialized. +- **KTD4. Treat the event schema as the primary privacy boundary.** Existing redaction helpers are a secondary defense. Reportable events are constructed from explicit allowlisted categorical fields rather than sanitized copies of arbitrary errors, configs, requests, tool args, or logs. +- **KTD5. Store identity and notice state in Caplets state, not config.** Config owns the user's opt-out preference. State owns random install ID, notice-shown marker, local delivery-health counters, and debug metadata. +- **KTD6. Suppress native-first telemetry until transparency exists.** OpenCode, Pi, and hidden daemon paths do not count hidden stderr or internal logs as user-visible notice. They may send only after CLI/status notice state exists or after a future native-visible notice channel is added. +- **KTD7. Allow CI/noninteractive collection without visible-notice mutation.** CI usage is product signal, but CI cannot satisfy a visible notice gate reliably. These events are tagged as CI/noninteractive and never mark the local notice as shown. +- **KTD8. Use official provider SDKs behind adapters.** Add `@sentry/node` and `posthog-node` to core behind lazy adapter factories. Do not build a Caplets relay or hand-rolled HTTP transport in the first release. +- **KTD9. Keep telemetry non-blocking and bounded.** Events are queued or dropped with local delivery-health accounting; repeated failures and CI loops are sampled or rate-limited. +- **KTD10. Make provider readiness an implementation gate, not a release afterthought.** Provider privacy settings, retention, ingestion monitoring, revocation, and decision-question readouts must be validated before broad instrumentation lands. + +## High-Level Technical Design + +```mermaid +flowchart TB + Source["CLI / native / MCP / daemon / Code Mode surface"] --> Context["Telemetry context"] + Context --> Resolver["state resolver"] + Resolver --> Disabled["disabled / suppressed"] + Resolver --> Notice["notice gate"] + Notice --> Disabled + Notice --> Builder["allowlisted event builder"] + Builder --> Debug["debug sink"] + Builder --> Providers["provider adapters"] + Providers --> PostHog["PostHog product events"] + Providers --> Sentry["Sentry reliability events"] + Providers --> Health["local delivery health"] +``` + +```mermaid +stateDiagram-v2 + [*] --> Disabled: env/config/test disables + [*] --> Suppressed: no visible notice and hidden native/daemon surface + [*] --> NeedsNotice: interactive user-visible surface + NeedsNotice --> Enabled: notice printed to stderr and recorded + [*] --> EnabledCI: CI/noninteractive allowed + Enabled --> [*]: emit allowlisted events + EnabledCI --> [*]: emit tagged events without notice mutation + Suppressed --> [*]: primary workflow continues + Disabled --> [*]: primary workflow continues +``` + +The shared telemetry layer has four jobs: + +- Resolve enabled/disabled/suppressed/debug state from env, config, execution context, notice state, and test detection. +- Create or read a random anonymous installation ID only when an event is eligible to send. +- Build product and reliability events from typed allowlists. +- Send through debug, PostHog, or Sentry adapters without blocking user workflows. + +## System-Wide Impact + +- **Configuration:** User config gains a top-level telemetry control and generated schema changes. Project config remains out of the control surface so repositories cannot silently override a user's privacy preference. +- **State lifecycle:** Caplets-owned state gains anonymous identity, notice, and delivery-health records. Disablement preserves identity by default; delete and rotate commands are explicit local state mutations and do not erase provider-side historical anonymous events. +- **CLI behavior:** Eligible commands may write one stderr notice before telemetry sends when stderr is an interactive TTY. Stdout, especially JSON output, remains a compatibility boundary and must not receive notice text or debug noise unless the command is explicitly `telemetry debug`. +- **Runtime behavior:** Serve, attach, daemon, MCP callbacks, native services, and Code Mode receive a telemetry context. Provider failures are swallowed after local health accounting, and primary tool execution results remain unchanged. +- **Privacy boundary:** Event builders become a new compatibility surface. Future telemetry additions must extend typed allowlists and tests rather than append arbitrary provider properties. +- **Provider operations:** Shipping SDKs and intake identifiers creates an operational dependency on provider privacy settings, retention, scrubbing, ingestion monitoring, and revocation playbooks. + +## Telemetry Event Boundaries + +The implementation may refine exact event names, but every event must remain inside these categories and property families. + +| Category | Provider | Examples | Allowed properties | +| ----------------- | -------------------------- | ----------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| Setup funnel | PostHog | init, setup, add, auth, remote login, doctor | package, version, surface, runtime mode, execution context, command family, outcome | +| Runtime lifecycle | PostHog | serve start, attach start, daemon install/start/status, native startup/reload | surface, runtime mode, execution context, integration, outcome, duration bucket | +| Tool activation | PostHog | first successful tool execution, progressive/direct operation family | surface, runtime mode, exposure mode, backend-family counts, operation family, outcome | +| Code Mode outcome | PostHog | CLI/MCP/native Code Mode run | surface, runtime scope, outcome, timeout bucket, duration bucket, diagnostic code categories, session created/reused, any caplet invoked | +| Reliability | Sentry | CLI/native/serve/attach/daemon failures | package, version, surface, runtime mode, command family, error code, diagnostic category, OS family, architecture, Node major | +| Delivery health | PostHog or local aggregate | provider init/drop/send failure counts | provider, reason category, count bucket, surface, execution context | + +Never include downstream tool names, Caplet IDs, raw command arguments, raw error messages, stack traces, URLs, hostnames, paths, code, logs, prompt/resource content, credentials, token values, or env values in provider payloads. + +Sentry grouping must use an explicit categorical fingerprint, such as package, surface, command family, runtime mode, error code, and diagnostic category. Adapter tests should snapshot final tags/fingerprint after local filtering and `beforeSend` so privacy controls do not erase the ability to group reliability pressure. + +## Origin Trace + +| Origin flow or acceptance example | Planned coverage | +| ------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| F1 first eligible CLI run; AE3, AE4 | U1 and U3 define notice state, stderr-only printing, and repeat suppression; U4 and U5 apply it to eligible CLI, serve, and attach paths. | +| F2 telemetry-disabled run; AE1, AE2, AE7 | U1 resolves env/config/test suppression before providers exist; U2 proves disabled/test/debug paths do not construct provider clients; U7 covers native startup. | +| F3 native integration usage; AE8, AE13 | U7 instruments OpenCode/Pi/native service usage and suppresses native-first telemetry unless visible notice state already exists. | +| F4 sanitized error capture; AE10, AE11 | U2 defines reliability allowlists and Sentry filtering; U4, U5, U6, and U7 emit sanitized failure categories from CLI, runtime, Code Mode, and native boundaries. | +| AE5 telemetry debug | U2 provides the debug sink; U3 exposes `caplets telemetry debug -- `. | +| AE6 CI context | U1 classifies CI/noninteractive usage; U4 and U5 propagate execution context into product events. | +| AE9 backend/exposure counts | U5 captures backend-family and exposure-mode counts from runtime/tool boundaries without Caplet IDs. | +| AE12, AE15, AE16, AE17, AE18 provider operations | U2 handles local delivery health and bounded transport; U9 establishes provider readiness and readout gates; U8 publishes final user-facing docs and release metadata. | + +## Implementation Units + +### U1. Add telemetry config, state paths, and resolver + +- **Goal:** Establish the shared local controls, identity, notice, and suppression model before any provider work. +- **Requirements:** R1, R2, R3, R5, R6, R7, R8, R13 +- **Dependencies:** None +- **Files:** `packages/core/src/config.ts`, `packages/core/src/config/paths.ts`, `schemas/caplets-config.schema.json`, `config.example.json`, `packages/core/src/telemetry/state.ts`, `packages/core/src/telemetry/context.ts`, `packages/core/src/telemetry/identity.ts`, `packages/core/src/telemetry/notice.ts`, `packages/core/test/config.test.ts`, `packages/core/test/config-validation.test.ts`, `packages/core/test/config-paths.test.ts`, `packages/core/test/telemetry-state.test.ts` +- **Approach:** Add strict top-level `telemetry?: boolean` parsing for user config and generated schema, but make it a global-only preference: project config must not override it. Implement this by reading or preserving the user telemetry preference before project merge and rejecting, warning on, or ignoring `telemetry` in project config with explicit tests. Add Caplets state paths for telemetry identity, notice, and delivery-health files using owner-only permissions where supported and atomic writes for identity/notice updates. Implement a resolver that returns `enabled`, `disabled`, `suppressed`, or `debug` with the deciding control and never initializes providers. Generate the anonymous ID as random state only when an enabled send path needs it; disabling telemetry does not delete the ID unless the user asks. +- **Execution note:** Start with failing tests for env precedence, user-config disablement, project-config non-override, config parse failure suppression, test-env suppression, native hidden-surface suppression, visible-notice predicate, CI identity scope, and state path resolution. +- **Patterns to follow:** Config schema source in `packages/core/src/config.ts`; XDG/platform path helpers in `packages/core/src/config/paths.ts`; config tests that use temporary `CAPLETS_CONFIG`. +- **Test scenarios:** `telemetry: false` in user config disables both providers; `telemetry` in project config cannot enable or disable telemetry; `CAPLETS_DISABLE_TELEMETRY=1` wins over config; invalid config suppresses telemetry rather than guessing; test env suppresses telemetry; CI is classified without notice mutation; CI uses a persisted random ID when Caplets state persists and an explicitly tagged ephemeral ID when state is unavailable; identity is random and stable; disable/re-enable preserves the ID; state paths are platform-correct and owner-only where supported. +- **Verification:** `pnpm --filter @caplets/core test -- test/config.test.ts test/config-validation.test.ts test/config-paths.test.ts test/telemetry-state.test.ts` and `pnpm schema:check`. + +### U2. Build event schemas, privacy guard, debug sink, and provider adapters + +- **Goal:** Create the allowlisted event pipeline before instrumenting product surfaces. +- **Requirements:** R9, R10, R11, R12, R13, R14, R16 +- **Dependencies:** U1 +- **Files:** `packages/core/package.json`, `pnpm-lock.yaml`, `packages/core/src/telemetry/events.ts`, `packages/core/src/telemetry/privacy.ts`, `packages/core/src/telemetry/providers.ts`, `packages/core/src/telemetry/posthog.ts`, `packages/core/src/telemetry/sentry.ts`, `packages/core/src/telemetry/debug.ts`, `packages/core/src/telemetry/delivery.ts`, `packages/core/src/telemetry/index.ts`, `packages/core/test/telemetry-events.test.ts`, `packages/core/test/telemetry-providers.test.ts`, `packages/core/test/telemetry-redaction.test.ts` +- **Approach:** Add official SDK dependencies behind lazy adapter factories only after an early go/no-go spike proves disabled-path imports, import side effects, transitive package weight, shutdown behavior, and final provider payloads fit the privacy and runtime constraints. If either SDK fails those gates, disable that adapter or substitute a minimal sanitized transport before broad instrumentation. Define typed product and reliability event builders that reject unknown event names and property keys in tests. Configure PostHog with anonymous captures, explicit GeoIP disabling, bounded queue settings, and shutdown behavior for short-lived processes. Configure Sentry with `sendDefaultPii: false`, no traces/profiles/log collection, categorical fingerprints/tags, `beforeSend` privacy filtering, and no unsanitized stack traces. Implement a debug sink that captures the exact sanitized would-send events without provider calls. +- **Execution note:** Implement provider adapters with injectable factories/transports so disabled/test/debug paths can assert no SDK initialization and so Sentry grouping can be verified after filtering. +- **Patterns to follow:** Existing `toSafeError` and `redactSecrets` helpers are backup sanitizers only; event builders should start from categorical inputs. +- **Test scenarios:** Unknown event/property keys fail tests; raw path/URL/hostname/env/token-shaped values are rejected or bucketed before provider calls; disabled/test/debug states do not import/construct clients; PostHog events include `$process_person_profile: false`; Sentry events never include raw stack/message/details; delivery failures increment local health without throwing. +- **Verification:** `pnpm --filter @caplets/core test -- test/telemetry-events.test.ts test/telemetry-providers.test.ts test/telemetry-redaction.test.ts`. + +### U3. Add telemetry CLI controls and stderr notice behavior + +- **Goal:** Give users visible controls and make the first eligible CLI event transparent. +- **Requirements:** R1, R2, R3, R4, R5, R6, R7, R8 +- **Dependencies:** U1, U2 +- **Files:** `packages/core/src/cli/commands.ts`, `packages/core/src/cli.ts`, `packages/core/src/cli/telemetry.ts`, `packages/core/test/cli.test.ts`, `packages/core/test/cli-completion.test.ts`, `packages/core/test/telemetry-cli.test.ts` +- **Approach:** Add the `telemetry` top-level command and subcommands. `status` reports enabled/disabled/suppressed/debug state, deciding control, notice state, anonymous ID presence, and delivery-health summary. `enable` and `disable` mutate the user config only. `delete-id` and `rotate-id` manage local identity state only; command output and docs must say they do not delete provider-side historical anonymous events and should point to retention policy. Long-running processes resolve the current identity at send time, or refresh provider identity when the identity state changes, so a separate `delete-id` or `rotate-id` does not leave daemon/native/serve processes sending under a stale cached ID. `debug -- ` runs a nested Caplets command with the debug sink and prints sanitized would-send events locally without providers. Eligible interactive commands print the first-run notice to stderr before the first event only when stderr is an interactive TTY or another tested visible channel is present; help/completion/version/status/debug and parse errors do not emit product telemetry. +- **Execution note:** Keep notice text stable enough for tests, but avoid asserting large help-output snapshots. +- **Patterns to follow:** Commander wiring and injected `CliIO` in `packages/core/src/cli.ts`; command registry in `packages/core/src/cli/commands.ts`; daemon/vault/doctor command test style. +- **Test scenarios:** Notice writes to stderr only; redirected stderr does not mark notice shown; JSON stdout for commands is not polluted; repeated eligible commands do not repeat notice; `telemetry status` explains env-disable precedence; `enable` does not override `CAPLETS_DISABLE_TELEMETRY=1`; `delete-id` and `rotate-id` update local state and running send paths observe the new state; `debug` prints only sanitized payloads and no provider calls; completions include telemetry subcommands. +- **Verification:** `pnpm --filter @caplets/core test -- test/telemetry-cli.test.ts test/cli.test.ts test/cli-completion.test.ts`. + +### U9. Establish provider readiness and readout gate + +- **Goal:** Prove provider configuration, event interpretation, and release gating before broad product instrumentation depends on them. +- **Requirements:** R14, R15, R16, R17 +- **Dependencies:** U2 +- **Files:** `docs/product/anonymous-telemetry.md`, `docs/product/telemetry-readout.md`, `docs/product/telemetry-provider-readiness.md`, `packages/core/src/telemetry/events.ts`, `packages/core/src/telemetry/providers.ts`, `packages/core/test/telemetry-providers.test.ts` +- **Approach:** Add a versioned provider-readiness artifact that records provider project/environment, intake identifiers, settings checked, owner, review date, retention, revocation procedure, ingestion-monitoring status, and the event/query mapping back to the brainstorm decision questions. This artifact is a release gate for packaging telemetry-enabled builds with intake identifiers. Establish the first saved-query/report contract before instrumenting every surface so event names and categorical fields prove useful before broad wiring. +- **Execution note:** This unit intentionally stays after provider adapter shape and before broad instrumentation; it does not automate provider project setup. +- **Patterns to follow:** Product docs under `docs/product/`; event allowlist and provider adapter tests from U2. +- **Test scenarios:** Provider-readiness doc names all launch-gate fields; readout doc maps every decision question to at least one allowlisted event family; provider tests snapshot the categorical properties required by the readout. +- **Verification:** `pnpm docs:check` and focused provider/event tests from U2. + +### U4. Instrument CLI setup, config, auth, remote, doctor, and operation flows + +- **Goal:** Capture setup funnel and CLI usage signals without changing command behavior. +- **Requirements:** R9, R10, R11, R12, R16 +- **Dependencies:** U3, U9 +- **Files:** `packages/core/src/cli.ts`, `packages/core/src/cli/code-mode.ts`, `packages/core/src/remote/*`, `packages/core/src/auth/*`, `packages/core/src/setup/*`, `packages/core/test/cli.test.ts`, `packages/core/test/cli-remote.test.ts`, `packages/core/test/doctor-cli.test.ts`, `packages/core/test/code-mode-cli.test.ts`, `packages/core/test/telemetry-cli-events.test.ts` +- **Approach:** Wrap command actions with a telemetry context that records command family, surface, runtime mode, execution context, outcome, and duration bucket. Emit setup milestones for init/setup/add/auth/remote login/doctor, and operation-family events for inspect/check/tools/search/describe/call/resources/prompts/complete. Capture sanitized Sentry reliability events for CLI-visible failures, including errors converted to exit codes. Do not emit raw args or values from config, env, URLs, paths, server IDs, profile names, tokens, or tool payloads. +- **Execution note:** Characterize existing exit-code and stdout/stderr behavior before adding telemetry wrappers around commands that produce JSON. +- **Patterns to follow:** Existing CLI action tests that pass `writeOut`, `writeErr`, `env`, `fetch`, and `setExitCode` fakes; `toSafeError` for user-facing JSON remains separate from telemetry payloads. +- **Test scenarios:** Setup funnel commands emit only categorical milestones; JSON-producing commands keep stdout exact; failing commands report sanitized reliability categories; debug mode captures would-send events; config/env/URL/path inputs never appear in payload snapshots; provider errors do not change exit codes. +- **Verification:** Focused CLI telemetry tests plus existing CLI test files touched by command wrappers. + +### U5. Instrument MCP serve, attach, daemon, and tool execution surfaces + +- **Goal:** Capture long-running runtime adoption and tool activation without treating hidden output as transparency. +- **Requirements:** R5, R6, R7, R9, R10, R11, R12, R16 +- **Dependencies:** U3, U9 +- **Files:** `packages/core/src/serve/index.ts`, `packages/core/src/serve/session.ts`, `packages/core/src/serve/native-session.ts`, `packages/core/src/serve/http.ts`, `packages/core/src/attach/server.ts`, `packages/core/src/daemon/*`, `packages/core/src/engine.ts`, `packages/core/test/serve-daemon.test.ts`, `packages/core/test/serve-http.test.ts`, `packages/core/test/attach-cli.test.ts`, `packages/core/test/native-remote.test.ts`, `packages/core/test/telemetry-runtime.test.ts` +- **Approach:** Emit startup/lifecycle events for foreground serve, attach, HTTP sessions, and daemon lifecycle commands. Instrument registered MCP tool callbacks and engine execution boundaries so progressive/direct tool outcomes and backend-family/exposure-mode counts are captured even when errors are converted to `errorResult`. Hidden daemon stderr and daemon logs do not mark notice as shown; daemon runtime telemetry sends only if a prior visible notice exists or the run is CI/noninteractive and allowed. +- **Execution note:** Do not use daemon logs, MCP request bodies, tool args, tool outputs, resource contents, prompt contents, or downstream names as telemetry source material. +- **Patterns to follow:** `CapletsMcpSession` tool registration callbacks; `CapletsEngine.execute`/`executeDirectTool` errorResult behavior; daemon service-management plan and tests for hidden runtime behavior. +- **Test scenarios:** `serve` prints first notice before first interactive event; `attach` does not repeat notice after notice state exists; hidden daemon runtime suppresses without notice; tool execution emits operation/exposure/backend categories only; engine-caught errors generate sanitized reliability categories; provider failure never breaks MCP callbacks. +- **Coordination note:** Serve and attach CLI wrappers instrumented in U4 must use the same telemetry context and event families as runtime/session callbacks here. +- **Verification:** Focused runtime telemetry tests plus existing serve/attach/daemon/native-remote tests. + +### U6. Instrument Code Mode outcomes with strict content exclusion + +- **Goal:** Measure Code Mode success, failure, timeout, duration, session reuse, and diagnostic categories without collecting code or run artifacts. +- **Requirements:** R9, R10, R11, R12 +- **Dependencies:** U2, U4, U5 +- **Files:** `packages/core/src/code-mode/runner.ts`, `packages/core/src/code-mode/tool.ts`, `packages/core/src/cli/code-mode.ts`, `packages/core/src/serve/session.ts`, `packages/core/src/native/service.ts`, `packages/core/test/code-mode-runner.test.ts`, `packages/core/test/code-mode-cli.test.ts`, `packages/core/test/code-mode-mcp.test.ts`, `packages/core/test/telemetry-code-mode.test.ts` +- **Approach:** Add a telemetry context to Code Mode callers and emit outcome events from the runner or runner boundary after sanitizing to categorical fields. Include runtime scope, outcome, timeout bucket, duration bucket, diagnostic codes, session created/reused category, and whether any Caplet was invoked. Exclude source code, logs, diagnostics messages, declaration hashes, trace IDs, run IDs, session IDs, recovery refs, tool args, tool outputs, and journal contents. +- **Execution note:** Code Mode already has redacted recovery/audit machinery; telemetry must not reuse those stored artifacts as payloads. +- **Patterns to follow:** `CodeModeRunEnvelope` meta/outcome structure; existing Code Mode runner, CLI, and MCP tests. +- **Test scenarios:** Successful, diagnostic-failure, thrown-error, timeout, and reused-session runs emit expected categorical payloads; payload snapshots never include code/logs/session/recovery IDs; debug mode prints sanitized Code Mode event; disabled/test states do not initialize providers. +- **Verification:** `pnpm --filter @caplets/core test -- test/telemetry-code-mode.test.ts test/code-mode-runner.test.ts test/code-mode-cli.test.ts test/code-mode-mcp.test.ts`. + +### U7. Instrument native service and integrations + +- **Goal:** Capture OpenCode and Pi usage through the same privacy and notice model as CLI and MCP surfaces. +- **Requirements:** R5, R6, R9, R10, R11, R12, R16 +- **Dependencies:** U1, U2, U5, U6 +- **Files:** `packages/core/src/native/service.ts`, `packages/core/src/native/options.ts`, `packages/core/src/native/remote.ts`, `packages/core/src/native.ts`, `packages/opencode/src/index.ts`, `packages/opencode/src/hooks.ts`, `packages/pi/src/index.ts`, `packages/core/test/native.test.ts`, `packages/core/test/native-remote.test.ts`, `packages/opencode/test/opencode.test.ts`, `packages/pi/test/pi.test.ts`, `packages/core/test/telemetry-native.test.ts` +- **Approach:** Thread telemetry context through native service creation, reload, tool listing, execution, and Code Mode service calls. Classify native integration, runtime mode, execution context, backend/exposure counts, and outcome categories. Suppress native-first telemetry unless visible notice state exists; keep integration-specific warnings/status displays separate from the durable telemetry notice unless they can be proven user-visible and test-covered. +- **Sequencing note:** Native lifecycle/tool telemetry can land after U5; native Code Mode propagation depends on U6. +- **Execution note:** Native options may include project roots, remote URLs, auth dirs, workspace selectors, and fetch functions; none of those values can enter telemetry payloads. +- **Patterns to follow:** `createNativeCapletsService` as the shared native boundary; OpenCode/Pi wrappers as thin integration adapters; `safeErrorMessage` style in Pi for user-facing output only. +- **Test scenarios:** Native startup disabled by env/config/test does not initialize providers; native-first run without notice suppresses product events; prior visible notice allows native categorical events; native execution failure sends sanitized reliability category; remote/cloud mode classification does not include URLs or workspace IDs. +- **Verification:** Native telemetry tests plus existing core/opencode/pi package tests. + +### U8. Add provider privacy checklist, usage readout, docs, and release metadata + +- **Goal:** Make telemetry understandable to users and useful to maintainers at launch. +- **Requirements:** R4, R12, R14, R15, R16, R17 +- **Dependencies:** U1, U2, U3, U4, U5, U6, U7, U9 +- **Files:** `README.md`, `packages/cli/README.md`, `packages/opencode/README.md`, `packages/pi/README.md`, `apps/docs/src/content/docs/install.mdx`, `apps/docs/src/content/docs/agent-integrations.mdx`, `apps/docs/src/content/docs/troubleshooting.mdx`, `docs/product/anonymous-telemetry.md`, `docs/product/telemetry-readout.md`, `.changeset/*` +- **Approach:** Document what Caplets collects, what it never collects, how to disable, how to inspect status, how to rotate/delete the local ID, and how debug mode works. State clearly that delete/rotate controls affect local anonymous identity only and do not erase provider-side historical anonymous events; provider retention controls historical data. Finalize user-facing docs from the U9 provider-readiness and readout artifacts, including PostHog project token handling, Sentry DSN handling, IP/GeoIP/privacy settings, retention, scrubbing, quota monitoring, ingestion alerts, and revocation procedure. Add saved-query/report guidance that maps collected events back to setup funnel, surface adoption, runtime investment, exposure mode, backend-family usage, and reliability questions. +- **Execution note:** Keep provider management tokens out of repository, package contents, docs examples, and test snapshots. +- **Patterns to follow:** Existing docs structure under `apps/docs/src/content/docs/`; generated docs checks; Changesets requirement for user-facing package behavior. +- **Test scenarios:** Docs mention stderr notice and both disable controls; docs state exact never-collected categories; docs check passes; package artifacts do not include private provider credentials; changeset exists for CLI/core/native integration packages. +- **Verification:** `pnpm docs:check`, `pnpm --filter caplets build`, `pnpm --filter @caplets/opencode build`, `pnpm --filter @caplets/pi build`, and `pnpm changeset status --since=origin/main` in PR context. + +## Scope Boundaries + +- This plan does not add a Caplets-hosted telemetry relay or self-hosted telemetry backend. +- This plan does not automate Sentry/PostHog project creation, dashboards, saved queries, retention settings, or privacy settings through provider APIs. +- This plan does not collect account, email, workspace, organization, host, path, URL, Caplet ID, prompt, config, tool payload, Code Mode source, log, stack, or raw error-message content. +- This plan does not make native integrations display a new visible telemetry notice unless the integration can provide a testable user-visible channel in the implementation slice. +- This plan does not change daemon install-time service config semantics except where needed to classify telemetry execution context and preserve hidden-output notice behavior. +- This plan does not attempt exact per-user analytics or paid-conversion attribution. + +## Risks & Dependencies + +- **Provider privacy drift:** Sentry/PostHog project settings can drift outside code. The provider checklist must be completed before release and reviewed on the same cadence as telemetry readouts. +- **Bundle/runtime weight:** Adding provider SDKs to core affects CLI and native packages. Lazy imports and disabled/test path assertions are required to keep disabled runs clean. +- **Notice ambiguity:** Hidden stderr and native integration surfaces are easy to misclassify. The resolver should default to suppressed when visibility is unknown. +- **Event overreach:** Feature pressure may tempt adding raw IDs or names later. Event allowlist tests must fail on unknown keys and suspicious string shapes. +- **Sentry stack defaults:** Sentry SDK defaults may include stack frames or request-like context. Adapter tests and `beforeSend` filtering must prove provider payloads remain categorical. +- **Daemon env persistence:** Install-time daemon env behavior can make per-process disablement subtle. `telemetry status` and docs should explain whether the running managed service inherited a disable setting. +- **Telemetry quality:** Fire-and-forget transport can hide delivery failures. Delivery-health events and provider ingestion monitoring are needed before using absence of events as product evidence. +- **Bun/runtime compatibility:** Core package metadata and docs may imply non-Node runtimes, while this plan selects Node SDKs and Node major-version context. U2 should confirm adapter behavior in every supported package runtime or gate telemetry to supported runtimes explicitly. + +## Provider Operations Launch Gates + +- Sentry DSN and PostHog project token are intake-only, environment-specific, and revocable; no provider management, read, or admin keys ship in code, docs, package artifacts, tests, or CI logs. +- PostHog project settings disable or scrub IP/geolocation capture where supported, and event captures set anonymous profile behavior explicitly. +- Sentry project settings enable server-side scrubbing, SDK configuration disables default PII, and adapter tests prove stack/message/details are not sent. +- Retention limits are recorded for both providers before release, with an owner and review cadence in `docs/product/anonymous-telemetry.md`. +- Provider quotas, ingestion errors, and repeated-event abuse controls are monitored or accounted for in `docs/product/telemetry-readout.md`. +- A revocation playbook exists for rotating shipped intake identifiers and validating that old identifiers no longer ingest data. + +## Sources / Research + +- Requirements: `docs/brainstorms/2026-06-24-anonymous-telemetry-requirements.md`. +- Product direction and vocabulary: `STRATEGY.md`, `CONCEPTS.md`. +- Config and schema source: `packages/core/src/config.ts`, `schemas/caplets-config.schema.json`, `config.example.json`. +- State path patterns: `packages/core/src/config/paths.ts`. +- CLI command wiring and injected IO: `packages/core/src/cli.ts`, `packages/core/src/cli/commands.ts`, `packages/core/src/cli/code-mode.ts`. +- Runtime and tool execution boundaries: `packages/core/src/engine.ts`, `packages/core/src/serve/index.ts`, `packages/core/src/serve/session.ts`, `packages/core/src/attach/server.ts`. +- Native integration boundaries: `packages/core/src/native/service.ts`, `packages/opencode/src/index.ts`, `packages/opencode/src/hooks.ts`, `packages/pi/src/index.ts`. +- Code Mode outcome sources: `packages/core/src/code-mode/runner.ts`, `packages/core/src/code-mode/tool.ts`. +- Existing safety helpers: `packages/core/src/errors.ts`, `packages/core/src/redaction.ts`. +- Daemon and hidden-runtime precedent: `docs/plans/2026-06-19-001-feat-caplets-daemon-service-plan.md`, `docs/solutions/architecture-patterns/native-daemon-service-management.md`. +- Code Mode privacy precedent: `docs/solutions/architecture-patterns/code-mode-repl-sessions.md`, `docs/plans/2026-06-17-002-feat-code-mode-repl-sessions-plan.md`. +- Sentry Node SDK docs: https://docs.sentry.io/platforms/javascript/guides/node/ +- Sentry options docs for `sendDefaultPii`, transport, and `beforeSend`: https://docs.sentry.io/platforms/javascript/configuration/options/ +- Sentry server-side scrubbing docs: https://docs.sentry.io/security-legal-pii/scrubbing/server-side-scrubbing/ +- PostHog Node SDK docs for batching, `capture`, `$process_person_profile`, `disableGeoip`, and shutdown: https://posthog.com/docs/libraries/node +- PostHog privacy controls docs: https://posthog.com/docs/product-analytics/privacy +- PostHog data-collection opt-out and IP controls docs: https://posthog.com/docs/privacy/data-collection +- PostHog troubleshooting docs for public project tokens versus private API keys: https://posthog.com/docs/product-analytics/troubleshooting diff --git a/docs/product/anonymous-telemetry.md b/docs/product/anonymous-telemetry.md new file mode 100644 index 00000000..a9e82a0f --- /dev/null +++ b/docs/product/anonymous-telemetry.md @@ -0,0 +1,17 @@ +# Anonymous Telemetry + +Caplets collects opt-out anonymous telemetry to understand setup friction, runtime adoption, integration usage, backend-family investment, exposure-mode usage, Code Mode outcomes, and reliability pressure. + +Telemetry is disabled when `CAPLETS_DISABLE_TELEMETRY=1` is set, when the user config has top-level `"telemetry": false`, or in test environments. Use `caplets telemetry status`, `caplets telemetry disable`, `caplets telemetry enable`, `caplets telemetry rotate-id`, and `caplets telemetry delete-id` to inspect or change local telemetry state. + +The first eligible interactive CLI run writes this notice to stderr only: + +```text +Caplets collects anonymous telemetry for product usage and reliability. Disable it with CAPLETS_DISABLE_TELEMETRY=1 or `caplets telemetry disable`. +``` + +Caplets never collects raw config, prompts, Code Mode code, tool arguments, tool outputs, logs, resource contents, prompt contents, file paths, URLs, hostnames, Caplet IDs, credentials, tokens, raw environment variables, raw error messages, or unsanitized stack traces. + +`delete-id` and `rotate-id` affect only the local anonymous installation ID. They do not delete provider-side historical anonymous events; provider retention settings govern historical data. + +Provider readiness, retention, scrubbing, revocation, quota, and ingestion-monitoring gates are tracked in `docs/product/telemetry-provider-readiness.md`. diff --git a/docs/product/telemetry-provider-readiness.md b/docs/product/telemetry-provider-readiness.md new file mode 100644 index 00000000..00c15d86 --- /dev/null +++ b/docs/product/telemetry-provider-readiness.md @@ -0,0 +1,46 @@ +# Telemetry Provider Readiness + +Version: 1 + +Status: not ready for broad release until every launch gate below is checked for the release environment. + +## Provider Project + +| Field | Value | +| ------------------------- | -------------------------------------- | +| Environment | production | +| PostHog project | TODO before release | +| PostHog intake identifier | project token only, no private API key | +| Sentry project | TODO before release | +| Sentry intake identifier | DSN only, no auth token | +| Owner | TODO before release | +| Review date | TODO before release | +| Review cadence | before every telemetry-enabled release | + +## Launch Gates + +| Gate | Required check | Status | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | +| Intake identifiers | PostHog token and Sentry DSN are environment-specific, revocable, and contain no management, read, or admin privileges. | TODO | +| Package artifacts | No provider management keys, private API keys, read tokens, admin tokens, or CI secrets are present in package contents, docs examples, tests, or logs. | TODO | +| PostHog IP and GeoIP | Project settings disable or scrub IP/geolocation capture where supported; SDK captures set `$geoip_disable: true` and `$process_person_profile: false`. | TODO | +| Sentry privacy | Project server-side scrubbing is enabled; SDK uses `sendDefaultPii: false`; adapter tests prove raw stack, message, request, breadcrumb, and extra data are stripped. | TODO | +| Retention | Retention limits for PostHog and Sentry are recorded and accepted by the owner. | TODO | +| Ingestion monitoring | Provider ingestion errors, quota pressure, and unexpected event spikes have an owner-visible monitoring path. | TODO | +| Revocation | A playbook exists to rotate shipped PostHog/Sentry intake identifiers and validate old identifiers no longer ingest data. | TODO | +| Delivery health | Local delivery-health counters are reviewed before interpreting missing events as missing usage. | TODO | +| Readout mapping | `docs/product/telemetry-readout.md` maps decision questions to allowlisted event families and properties. | TODO | + +## Revocation Playbook + +1. Create replacement PostHog project token and Sentry DSN in the release environment. +2. Build a patch release with the replacement intake identifiers or environment configuration. +3. Revoke or disable the old identifiers in provider settings. +4. Validate that old identifiers no longer ingest events. +5. Check delivery-health counters and provider ingestion dashboards for the first release window after rotation. + +## Release Gate + +Telemetry-enabled packaging must not ship with intake identifiers until this document has non-TODO owner, review date, retention, ingestion-monitoring, and revocation entries for the target environment. + +The GitHub Actions release workflow requires the release environment to define `CAPLETS_POSTHOG_TOKEN` and `CAPLETS_SENTRY_DSN` as repository or environment secrets. These values are passed only to the release job environment and must remain runtime intake identifiers, not hardcoded package contents. diff --git a/docs/product/telemetry-readout.md b/docs/product/telemetry-readout.md new file mode 100644 index 00000000..4b2ef9b7 --- /dev/null +++ b/docs/product/telemetry-readout.md @@ -0,0 +1,25 @@ +# Telemetry Readout + +This readout maps anonymous event families back to the product questions that motivated telemetry. Saved provider queries should use only the allowlisted event names and categorical properties in `packages/core/src/telemetry/events.ts`. + +## Decision Questions + +| Question | Event families | Required properties | +| --------------------------------------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Where does setup fail? | `caplets_setup_milestone`, `caplets_cli_command`, `caplets_reliability_error` | `command_family`, `outcome`, `duration_bucket`, `error_code`, `diagnostic_category` | +| Which surfaces are active? | `caplets_cli_command`, `caplets_runtime_lifecycle`, `caplets_tool_activation` | `surface`, `execution_context`, `runtime_mode`, `outcome` | +| Is local, remote, or cloud runtime worth more investment? | `caplets_runtime_lifecycle`, `caplets_tool_activation`, `caplets_reliability_error` | `runtime_mode`, `surface`, `outcome`, `error_code` | +| Are native integrations used? | `caplets_runtime_lifecycle`, `caplets_tool_activation`, `caplets_code_mode_outcome` | `surface`, `integration`, `runtime_mode`, `outcome` | +| Which exposure modes are used? | `caplets_tool_activation` | `exposure_mode`, `direct_count`, `progressive_count`, `code_mode_count`, `operation_family` | +| Which backend families deserve investment? | `caplets_tool_activation`, `caplets_runtime_lifecycle` | `backend_mcp_count`, `backend_openapi_count`, `backend_google_discovery_count`, `backend_graphql_count`, `backend_http_count`, `backend_cli_count`, `backend_caplets_count` | +| Is Code Mode succeeding? | `caplets_code_mode_outcome`, `caplets_reliability_error` | `outcome`, `duration_bucket`, `timeout_bucket`, `session_category`, `any_caplet_invoked`, `diagnostic_category` | +| What reliability pressure is highest? | `caplets_reliability_error`, `caplets_delivery_health` | `surface`, `runtime_mode`, `command_family`, `error_code`, `diagnostic_category`, `provider`, `reason`, `count_bucket` | + +## Saved Query Contract + +- Setup funnel: count `caplets_setup_milestone` by `command_family` and `outcome`. +- Surface adoption: count `caplets_runtime_lifecycle` by `surface`, `runtime_mode`, and `execution_context`. +- First activation: count `caplets_tool_activation` successes by `surface`, `operation_family`, and `exposure_mode`. +- Code Mode outcomes: count `caplets_code_mode_outcome` by `outcome`, `timeout_bucket`, and `session_category`. +- Reliability: count `caplets_reliability_error` by `surface`, `command_family`, `error_code`, and `diagnostic_category`. +- Delivery health: count `caplets_delivery_health` by `provider`, `reason`, and `count_bucket`. diff --git a/package.json b/package.json index d4200988..88947de2 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "schema:check": "tsx ./scripts/generate-config-schema.ts --check", "schema:generate": "tsx ./scripts/generate-config-schema.ts", "test": "vitest run", + "telemetry:check-release-env": "tsx ./scripts/check-telemetry-release-env.ts", "typecheck": "tsgo --noEmit && turbo typecheck", "verify": "pnpm format:check && pnpm lint && pnpm code-mode:check-api && pnpm schema:check && pnpm docs:check && pnpm typecheck && pnpm test && pnpm benchmark:check && pnpm build", "version-packages": "changeset version && oxlint --fix --quiet && oxfmt --write ." diff --git a/packages/core/package.json b/packages/core/package.json index f3ffa20a..21e6a8ec 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -98,6 +98,7 @@ "@hono/mcp": "^0.3.0", "@hono/node-server": "^2.0.4", "@modelcontextprotocol/sdk": "^1.29.0", + "@sentry/node": "^10.60.0", "@ungap/structured-clone": "^1.3.1", "ajv": "^8.20.0", "commander": "^15.0.0", @@ -105,6 +106,7 @@ "graphql": "^16.14.2", "headers-polyfill": "^5.0.1", "hono": "^4.12.25", + "posthog-node": "^4.18.0", "quickjs-emscripten": "^0.32.0", "typescript": "^6.0.3", "vfile": "^6.0.3", diff --git a/packages/core/src/cli.ts b/packages/core/src/cli.ts index b371df3a..c8074d44 100644 --- a/packages/core/src/cli.ts +++ b/packages/core/src/cli.ts @@ -1,5 +1,7 @@ import { Command, CommanderError, Option } from "commander"; import { Buffer } from "node:buffer"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { arch, platform } from "node:os"; import { dirname, join } from "node:path"; import { createInterface } from "node:readline/promises"; import { Writable } from "node:stream"; @@ -113,8 +115,26 @@ import { type DaemonOperationOptions, } from "./daemon"; import { resolveServeOptions, serveResolvedCaplets, type ServeOptions } from "./serve"; -import { DEFAULT_AUTH_DIR } from "./config/paths"; +import { DEFAULT_AUTH_DIR, defaultTelemetryStateDir } from "./config/paths"; import { appendBasePath } from "./server/options"; +import { + deleteTelemetryIdentity, + buildProductTelemetryEvent, + buildReliabilityTelemetryEvent, + createTelemetryDispatcher, + durationBucket, + maybePrintTelemetryNotice, + readTelemetryDeliveryHealth, + readTelemetryIdentity, + readTelemetryNotice, + resolveTelemetryState, + rotateTelemetryIdentity, + TelemetryDebugSink, + type CommandFamily, + type DiagnosticCategory, + type TelemetryDispatcher, + type TelemetrySurface, +} from "./telemetry"; import { FileVaultStore, VAULT_MAX_VALUE_BYTES, validateVaultKeyName } from "./vault"; import type { VaultAccessGrantFilter } from "./vault"; @@ -137,6 +157,9 @@ type CliIO = { signal?: AbortSignal; projectBindingWebSocketFactory?: ProjectBindingWebSocketFactory; authDir?: string; + telemetryStateDir?: string; + stderrIsTTY?: boolean; + telemetryDebugSink?: TelemetryDebugSink; version?: string; setExitCode?: (code: number) => void; serve?: (options: ServeOptions) => Promise; @@ -147,14 +170,44 @@ type CliIO = { }; export async function runCli(args: string[], io: CliIO = {}): Promise { - const program = createProgram(io); + let observedExitCode = 0; + const wrappedIo: CliIO = { + ...io, + setExitCode: (code) => { + observedExitCode = code; + if (io.setExitCode) { + io.setExitCode(code); + } else { + process.exitCode = code; + } + }, + }; + const program = createProgram(wrappedIo); + const trackedCommand = telemetryCommandFamilyFromArgs(args); + const startedAt = Date.now(); + const telemetryContext = telemetryContextForIo(wrappedIo); + const dispatcher = createTelemetryDispatcher({ + stateDir: telemetryContext.stateDir, + }); try { if (args.length === 0) { program.outputHelp(); return; } await program.parseAsync(["node", "caplets", ...args]); + if (trackedCommand) { + await captureCliTelemetry(telemetryContext, { + debugSink: wrappedIo.telemetryDebugSink, + dispatcher, + commandFamily: trackedCommand.commandFamily, + surface: trackedCommand.surface, + outcome: observedExitCode === 0 ? "success" : "failure", + startedAt, + }); + } } catch (error) { + let normalizedError = error; + let captureProductEvent = true; if (error instanceof CommanderError) { if ( error.code === "commander.helpDisplayed" || @@ -163,9 +216,24 @@ export async function runCli(args: string[], io: CliIO = {}): Promise { ) { return; } - throw new CapletsError("REQUEST_INVALID", error.message); + normalizedError = new CapletsError("REQUEST_INVALID", error.message); + captureProductEvent = false; } - throw error; + if (trackedCommand) { + await captureCliTelemetry(telemetryContext, { + debugSink: wrappedIo.telemetryDebugSink, + dispatcher, + commandFamily: trackedCommand.commandFamily, + surface: trackedCommand.surface, + outcome: "failure", + startedAt, + error: normalizedError, + productEvent: captureProductEvent, + }); + } + throw normalizedError; + } finally { + await dispatcher.shutdown(); } } @@ -256,6 +324,266 @@ const HIDDEN_INPUT_PROMPT_LABELS = { export const readHiddenInputForTest = readHiddenInput; +type TelemetryCliContext = { + env: NodeJS.ProcessEnv | Record; + configPath: string | undefined; + projectConfigPath: string; + stateDir: string; + stderrIsTTY: boolean; + writeErr: (value: string) => void; +}; + +function telemetryContextForIo(io: CliIO): TelemetryCliContext { + const env = io.env ?? process.env; + return { + env, + configPath: envConfigPath(env), + projectConfigPath: envProjectConfigPath(env), + stateDir: io.telemetryStateDir ?? defaultTelemetryStateDir(env), + stderrIsTTY: io.stderrIsTTY ?? process.stderr.isTTY === true, + writeErr: io.writeErr ?? ((value: string) => process.stderr.write(value)), + }; +} + +function telemetryCommandFamilyFromArgs( + args: string[], +): { commandFamily: CommandFamily; surface: TelemetrySurface } | undefined { + const command = args[0]; + if ( + command === undefined || + command === "--help" || + command === "-h" || + command === "--version" || + command === "-V" || + command === cliCommands.telemetry || + command === cliCommands.completion || + command === cliCommands.completeHidden + ) { + return undefined; + } + if (command === cliCommands.serve) return { commandFamily: "serve", surface: "serve" }; + if (command === cliCommands.attach) return { commandFamily: "attach", surface: "attach" }; + if (command === cliCommands.daemon) return { commandFamily: "daemon", surface: "daemon" }; + if (command === cliCommands.codeMode) { + return { commandFamily: "code_mode", surface: "code_mode" }; + } + if (command === cliCommands.setup) return { commandFamily: "setup", surface: "cli" }; + if (command === cliCommands.init) return { commandFamily: "init", surface: "cli" }; + if (command === cliCommands.install) return { commandFamily: "install", surface: "cli" }; + if (command === cliCommands.add) return { commandFamily: "add", surface: "cli" }; + if (command === cliCommands.doctor) return { commandFamily: "doctor", surface: "cli" }; + if (command === cliCommands.auth) return { commandFamily: "auth", surface: "cli" }; + if (command === cliCommands.remote || command === cliCommands.cloud) { + return { commandFamily: "remote", surface: "cli" }; + } + if ( + command === cliCommands.inspect || + command === cliCommands.checkBackend || + command === cliCommands.listTools || + command === cliCommands.searchTools || + command === cliCommands.getTool || + command === cliCommands.callTool + ) { + return { commandFamily: "tools", surface: "cli" }; + } + if ( + command === cliCommands.listResources || + command === cliCommands.searchResources || + command === cliCommands.listResourceTemplates || + command === cliCommands.readResource + ) { + return { commandFamily: "resources", surface: "cli" }; + } + if ( + command === cliCommands.listPrompts || + command === cliCommands.searchPrompts || + command === cliCommands.getPrompt + ) { + return { commandFamily: "prompts", surface: "cli" }; + } + if (command === cliCommands.complete) return { commandFamily: "complete", surface: "cli" }; + return { commandFamily: "unknown", surface: "cli" }; +} + +function telemetryConfigForCli(context: TelemetryCliContext): Pick { + try { + return loadConfigWithSources(context.configPath, context.projectConfigPath, { + vaultResolver: vaultBootstrapResolver, + }).config; + } catch (error) { + if (error instanceof CapletsError && error.code !== "CONFIG_INVALID") { + return {}; + } + return { telemetry: false }; + } +} + +function maybePrintCliTelemetryNotice( + context: TelemetryCliContext, + surface: TelemetrySurface, +): void { + const state = resolveTelemetryState({ + config: telemetryConfigForCli(context), + env: context.env, + stateDir: context.stateDir, + surface, + visibility: "visible", + }); + if (state.status !== "enabled" || state.notice.shown) return; + maybePrintTelemetryNotice({ + stateDir: context.stateDir, + surface, + stderrIsTTY: context.stderrIsTTY, + writeErr: context.writeErr, + }); +} + +async function captureCliTelemetry( + context: TelemetryCliContext, + options: { + debugSink?: TelemetryDebugSink | undefined; + dispatcher?: TelemetryDispatcher | undefined; + commandFamily: CommandFamily; + surface?: TelemetrySurface | undefined; + outcome: "success" | "failure"; + startedAt: number; + error?: unknown; + productEvent?: boolean | undefined; + }, +): Promise { + if (options.productEvent !== false) { + maybePrintCliTelemetryNotice(context, options.surface ?? "cli"); + } + const state = resolveTelemetryState({ + config: telemetryConfigForCli(context), + env: context.env, + stateDir: context.stateDir, + surface: options.surface ?? "cli", + visibility: "visible", + debug: context.env.CAPLETS_TELEMETRY_DEBUG === "1", + }); + if (state.status !== "enabled" && state.status !== "debug") return; + const identity = + state.status === "debug" + ? readTelemetryIdentity({ stateDir: context.stateDir, create: false }) + : (state.identity ?? readTelemetryIdentity({ stateDir: context.stateDir, create: true })); + if (options.productEvent !== false) { + const product = buildProductTelemetryEvent({ + name: "caplets_cli_command", + distinctId: identity.id, + properties: { + package: "@caplets/core", + version: packageJsonVersion, + surface: options.surface ?? "cli", + runtime_mode: runtimeModeForEnv(context.env), + execution_context: state.executionContext, + command_family: options.commandFamily, + outcome: options.outcome, + duration_bucket: durationBucket(Date.now() - options.startedAt), + }, + }); + if (state.status === "debug") { + options.debugSink?.capture("debug", product); + } else { + await ( + options.dispatcher ?? createTelemetryDispatcher({ stateDir: context.stateDir }) + ).capture(state, product); + } + } + + if (options.outcome !== "failure") return; + const reliability = buildReliabilityTelemetryEvent({ + name: "caplets_reliability_error", + properties: { + package: "@caplets/core", + version: packageJsonVersion, + surface: options.surface ?? "cli", + runtime_mode: runtimeModeForEnv(context.env), + command_family: options.commandFamily, + error_code: errorCodeForTelemetry(options.error), + diagnostic_category: diagnosticCategoryForError(options.error), + os_family: platform(), + arch: arch(), + node_major: Number(process.versions.node.split(".")[0] ?? 0), + }, + }); + if (state.status === "debug") { + options.debugSink?.capture("debug", reliability); + return; + } + await (options.dispatcher ?? createTelemetryDispatcher({ stateDir: context.stateDir })).capture( + state, + reliability, + ); +} + +function readUserConfigObject(path: string): Record { + if (!existsSync(path)) return {}; + try { + const parsed = JSON.parse(readFileSync(path, "utf8")); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + } catch (error) { + throw new CapletsError("CONFIG_INVALID", `Caplets config at ${path} is not valid JSON`, error); + } +} + +function runtimeModeForEnv(env: NodeJS.ProcessEnv | Record) { + const mode = env.CAPLETS_MODE; + return mode === "remote" || mode === "cloud" || mode === "local" ? mode : "unknown"; +} + +function errorCodeForTelemetry(error: unknown): string { + if (error instanceof CapletsError) return error.code; + return "UNKNOWN"; +} + +function diagnosticCategoryForError(error: unknown): DiagnosticCategory { + if (!(error instanceof CapletsError)) return "unknown"; + if (error.code.startsWith("CONFIG")) return "config"; + if (error.code.startsWith("AUTH")) return "auth"; + if (error.code.includes("NETWORK") || error.code.includes("UNAVAILABLE")) return "network"; + if (error.code.includes("VALID") || error.code.includes("REQUEST")) return "validation"; + return "runtime"; +} + +function writeTelemetryConfig(path: string, enabled: boolean): void { + const config = { + ...readUserConfigObject(path), + $schema: "https://caplets.dev/config.schema.json", + telemetry: enabled, + }; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 }); +} + +function formatTelemetryStatus(context: TelemetryCliContext): string { + const config = telemetryConfigForCli(context); + const state = resolveTelemetryState({ + config, + env: context.env, + stateDir: context.stateDir, + surface: "cli", + visibility: "visible", + createIdentity: false, + }); + const notice = readTelemetryNotice({ stateDir: context.stateDir }); + const identity = readTelemetryIdentity({ stateDir: context.stateDir, create: false }); + const health = readTelemetryDeliveryHealth({ stateDir: context.stateDir }); + const lines = [ + `Telemetry: ${state.status}`, + `Decision: ${state.decider}`, + `Config: ${config.telemetry === false ? "disabled" : "enabled"}`, + `Environment: ${context.env.CAPLETS_DISABLE_TELEMETRY === "1" ? "disabled" : "enabled"}`, + `Notice shown: ${notice.shown ? `yes (${notice.surface})` : "no"}`, + `Anonymous ID: ${identity.kind === "stable" ? "present" : "not stored"}`, + `Delivery health: ${Object.keys(health).length === 0 ? "none" : JSON.stringify(health)}`, + "Disable with CAPLETS_DISABLE_TELEMETRY=1 or `caplets telemetry disable`.", + ]; + return `${lines.join("\n")}\n`; +} + function remoteProfileStore( authDir: string | undefined, env: NodeJS.ProcessEnv | Record, @@ -968,6 +1296,16 @@ export function createProgram(io: CliIO = {}): Command { const writeErr = io.writeErr ?? ((value: string) => process.stderr.write(value)); const env = io.env ?? process.env; const currentConfigPath = () => envConfigPath(env); + const telemetryContext = (): TelemetryCliContext => ({ + env, + configPath: currentConfigPath(), + projectConfigPath: envProjectConfigPath(env), + stateDir: io.telemetryStateDir ?? defaultTelemetryStateDir(env), + stderrIsTTY: io.stderrIsTTY ?? process.stderr.isTTY === true, + writeErr, + }); + const printTelemetryNotice = (surface: TelemetrySurface) => + maybePrintCliTelemetryNotice(telemetryContext(), surface); const setExitCode = io.setExitCode ?? ((code: number) => { @@ -1051,6 +1389,78 @@ export function createProgram(io: CliIO = {}): Command { if (suggestions.length > 0) writeOut(`${suggestions.join("\n")}\n`); }); + const telemetry = program + .command(cliCommands.telemetry) + .description("Inspect and control anonymous Caplets telemetry."); + + telemetry + .command("status") + .description("Show anonymous telemetry status.") + .action(() => { + writeOut(formatTelemetryStatus(telemetryContext())); + }); + + telemetry + .command("enable") + .description("Enable anonymous telemetry in the user config.") + .action(() => { + const path = resolveConfigPath(currentConfigPath()); + writeTelemetryConfig(path, true); + writeOut(`Enabled anonymous telemetry in ${path}.\n`); + if (env.CAPLETS_DISABLE_TELEMETRY === "1") { + writeOut("CAPLETS_DISABLE_TELEMETRY=1 still disables telemetry for this process.\n"); + } + }); + + telemetry + .command("disable") + .description("Disable anonymous telemetry in the user config.") + .action(() => { + const path = resolveConfigPath(currentConfigPath()); + writeTelemetryConfig(path, false); + writeOut(`Disabled anonymous telemetry in ${path}.\n`); + }); + + telemetry + .command("delete-id") + .description("Delete the local anonymous telemetry ID.") + .action(() => { + deleteTelemetryIdentity({ stateDir: telemetryContext().stateDir }); + writeOut( + "Deleted the local anonymous telemetry ID. This does not delete provider-side historical anonymous events; provider retention controls historical data.\n", + ); + }); + + telemetry + .command("rotate-id") + .description("Rotate the local anonymous telemetry ID.") + .action(() => { + rotateTelemetryIdentity({ stateDir: telemetryContext().stateDir }); + writeOut( + "Rotated the local anonymous telemetry ID. This does not delete provider-side historical anonymous events; provider retention controls historical data.\n", + ); + }); + + telemetry + .command("debug") + .description("Run a Caplets command with local telemetry debug output.") + .allowUnknownOption(true) + .argument("[args...]", "Caplets command and arguments after --") + .action(async (args: string[]) => { + const nestedArgs = args[0] === "--" ? args.slice(1) : args; + const sink = new TelemetryDebugSink(); + if (nestedArgs.length > 0) { + await runCli(nestedArgs, { + ...io, + env: { ...env, CAPLETS_TELEMETRY_DEBUG: "1" }, + telemetryDebugSink: sink, + writeOut, + writeErr, + }); + } + writeOut(`${JSON.stringify({ telemetryDebug: sink.toJSON() }, null, 2)}\n`); + }); + const codeMode = program .command(cliCommands.codeMode) .description("Run, inspect, and debug Caplets Code Mode.") @@ -1163,6 +1573,7 @@ export function createProgram(io: CliIO = {}): Command { allowUnauthenticatedHttp?: boolean; trustProxy?: boolean; }) => { + printTelemetryNotice("serve"); const resolved = resolveServeOptions(options); const configPath = currentConfigPath(); const runner = @@ -1173,6 +1584,11 @@ export function createProgram(io: CliIO = {}): Command { { ...(configPath ? { configPath } : {}), ...(io.authDir ? { authDir: io.authDir } : {}), + telemetryEnv: env, + telemetryStateDir: io.telemetryStateDir ?? defaultTelemetryStateDir(env), + telemetrySurface: "serve", + telemetryVisibility: "visible", + telemetryRuntimeMode: runtimeModeForEnv(env), }, writeErr, )); @@ -1203,6 +1619,7 @@ export function createProgram(io: CliIO = {}): Command { addDaemonInstallOptions( daemon.command("install").description("Install or update the default Caplets daemon."), ).action(async (options: DaemonInstallCommandOptions) => { + printTelemetryNotice("daemon"); const prompt = options.json ? undefined : createSetupPromptHandle(io, writeOut); try { const result = await installDaemon(daemonInstallOptions(options), { @@ -1232,6 +1649,7 @@ export function createProgram(io: CliIO = {}): Command { .option("--purge", "remove daemon config, state, and logs") .option("--dry-run", "preview uninstall actions without mutation"), ).action(async (options: { json?: boolean; purge?: boolean; dryRun?: boolean }) => { + printTelemetryNotice("daemon"); const result = await uninstallDaemon( { purge: options.purge === true, dryRun: options.dryRun === true }, daemonOptions(), @@ -1245,6 +1663,7 @@ export function createProgram(io: CliIO = {}): Command { addJsonOption(daemon.command("start").description("Start the default Caplets daemon.")).action( async (options: DaemonCommandOptions) => { + printTelemetryNotice("daemon"); const result = await startDaemon(daemonOptions()); if (options.json) { writeOut(`${JSON.stringify(result, null, 2)}\n`); @@ -1259,6 +1678,7 @@ export function createProgram(io: CliIO = {}): Command { addJsonOption( daemon.command("restart").description("Restart the default Caplets daemon."), ).action(async (options: DaemonCommandOptions) => { + printTelemetryNotice("daemon"); const result = await restartDaemon(daemonOptions()); if (options.json) { writeOut(`${JSON.stringify(result, null, 2)}\n`); @@ -1269,6 +1689,7 @@ export function createProgram(io: CliIO = {}): Command { addJsonOption(daemon.command("stop").description("Stop the default Caplets daemon.")).action( async (options: DaemonCommandOptions) => { + printTelemetryNotice("daemon"); const result = await stopDaemon(daemonOptions()); if (options.json) { writeOut(`${JSON.stringify(result, null, 2)}\n`); @@ -1281,6 +1702,7 @@ export function createProgram(io: CliIO = {}): Command { addJsonOption( daemon.command("status").description("Show the default Caplets daemon status."), ).action(async (options: DaemonCommandOptions) => { + printTelemetryNotice("daemon"); const status = await daemonStatus(daemonOptions()); if (options.json) { writeOut(`${JSON.stringify(status, null, 2)}\n`); @@ -1297,6 +1719,7 @@ export function createProgram(io: CliIO = {}): Command { .option("--tail ", "show the last number of lines") .option("--stream ", "log stream: stdout, stderr, or all"), ).action(async (options: DaemonLogsCommandOptions) => { + printTelemetryNotice("daemon"); const tail = options.tail === undefined ? 10 : parseNonNegativeInteger(options.tail, "--tail"); const stream = parseLogStream(options.stream); if (options.follow) { @@ -1376,6 +1799,7 @@ export function createProgram(io: CliIO = {}): Command { projectRoot?: string; }, ) => { + printTelemetryNotice("attach"); try { rejectAttachHttpServeFlags(options); const remoteUrl = attachRemoteUrlFromArgs(url, options.remoteUrl); @@ -2022,6 +2446,7 @@ export function createProgram(io: CliIO = {}): Command { format?: SetupFormat; }, ) => { + printTelemetryNotice("cli"); const setupOptions: SetupOptions = { ...options, env }; if (io.runSetupCommand) setupOptions.runCommand = io.runSetupCommand; if (!integration) { @@ -2406,6 +2831,7 @@ export function createProgram(io: CliIO = {}): Command { capletIds: string[], options: MutationTargetOptions & { force?: boolean }, ) => { + printTelemetryNotice("cli"); const target = parseMutationTarget(options); if (target === "remote") { const remote = requireRemoteClientForTarget(io); @@ -4157,6 +4583,11 @@ async function executeLocalOperation( ...(io.authDir ? { authDir: io.authDir } : {}), watch: false, writeErr: io.writeErr, + telemetryEnv: io.env ?? process.env, + telemetryStateDir: defaultTelemetryStateDir(io.env ?? process.env), + telemetrySurface: "cli", + telemetryVisibility: "visible", + telemetryRuntimeMode: runtimeModeForEnv(io.env ?? process.env), ...(config ? { configLoader: () => config } : {}), }); try { diff --git a/packages/core/src/cli/code-mode.ts b/packages/core/src/cli/code-mode.ts index b9e6ddce..ac3d147a 100644 --- a/packages/core/src/cli/code-mode.ts +++ b/packages/core/src/cli/code-mode.ts @@ -1,5 +1,6 @@ import { readFileSync } from "node:fs"; import { resolve } from "node:path"; +import { defaultTelemetryStateDir } from "../config/paths"; import { createNativeCapletsService } from "../native/service"; import { codeModeDeclarationHash, generateCodeModeDeclarations } from "../code-mode/declarations"; import { runCodeMode } from "../code-mode/runner"; @@ -34,6 +35,11 @@ export async function runCodeModeCli(options: CodeModeCliOptions): Promise ...(options.configPath ? { configPath: options.configPath } : {}), ...(options.projectConfigPath ? { projectConfigPath: options.projectConfigPath } : {}), ...(options.authDir ? { authDir: options.authDir } : {}), + telemetryEnv: options.env as NodeJS.ProcessEnv | undefined, + telemetryStateDir: defaultTelemetryStateDir(options.env), + telemetrySurface: "code_mode", + telemetryVisibility: "visible", + telemetryRuntimeMode: runtimeScope(options.env) === "local" ? "local" : "unknown", }); try { if (options.sessionId !== undefined) { @@ -57,12 +63,19 @@ export async function runCodeModeCli(options: CodeModeCliOptions): Promise return; } const code = await readCodeModeCliCode(options); + const started = Date.now(); const result = await runCodeMode({ code, - service: service.codeModeService?.() ?? service, ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), + service: service.codeModeService?.() ?? service, runtimeScope: "cli-one-shot", }); + void service + .captureCodeModeOutcome?.(result, { + started, + ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), + }) + .catch(() => undefined); if (options.json) { options.writeOut(`${JSON.stringify(result, null, 2)}\n`); } else if (result.ok) { diff --git a/packages/core/src/cli/commands.ts b/packages/core/src/cli/commands.ts index b29f84b4..7855338d 100644 --- a/packages/core/src/cli/commands.ts +++ b/packages/core/src/cli/commands.ts @@ -33,6 +33,7 @@ export const cliCommands = { config: "config", auth: "auth", vault: "vault", + telemetry: "telemetry", } as const; export const topLevelCommandNames = [ @@ -65,6 +66,7 @@ export const topLevelCommandNames = [ cliCommands.config, cliCommands.auth, cliCommands.vault, + cliCommands.telemetry, cliCommands.completion, ] as const; @@ -78,6 +80,7 @@ export const cliSubcommands = { [cliCommands.config]: ["path", "paths"], [cliCommands.daemon]: ["install", "uninstall", "start", "restart", "stop", "status", "logs"], [cliCommands.setup]: ["codex", "claude-code", "opencode", "pi", "mcp-client"], + [cliCommands.telemetry]: ["status", "enable", "disable", "delete-id", "rotate-id", "debug"], [cliCommands.vault]: ["set", "get", "list", "delete", "access"], } as const satisfies Record; diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index 073e4ddb..c776010d 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -22,13 +22,16 @@ export { DEFAULT_AUTH_DIR, DEFAULT_COMPLETION_CACHE_DIR, DEFAULT_CONFIG_PATH, + DEFAULT_TELEMETRY_STATE_DIR, PROJECT_CONFIG_FILE, defaultCacheBaseDir, defaultCompletionCacheDir, + defaultTelemetryStateDir, resolveCapletsRoot, resolveConfigPath, resolveProjectCapletsRoot, resolveProjectConfigPath, + resolveTelemetryStateDir, } from "./config/paths"; export type RemoteAuthConfig = @@ -335,6 +338,7 @@ export type CompletionConfig = { export type CapletsConfig = { version: 1; + telemetry?: boolean | undefined; options: CapletsOptions; namespaceAliases: NamespaceAliasesConfig; mcpServers: Record; @@ -1137,6 +1141,7 @@ type ConfigSchemaHttpApiValue = z.infer; type ConfigSchemaCliToolsValue = z.infer; type ConfigSchemaCapletSetValue = z.infer; type ConfigInput = { + telemetry?: unknown; namespaceAliases?: unknown; mcpServers?: Record; openapiEndpoints?: Record; @@ -1191,6 +1196,10 @@ function configSchemaFor( .max(50) .default(50) .describe("Maximum accepted search_tools limit."), + telemetry: z + .boolean() + .optional() + .describe("Set false to disable anonymous Caplets telemetry for this user config."), completion: z .object({ discoveryTimeoutMs: z.number().int().positive().default(750), @@ -2479,6 +2488,7 @@ function mergeConfigInputs(...inputs: Array): ConfigInp merged = { ...merged, ...input, + telemetry: input.telemetry === undefined ? merged?.telemetry : input.telemetry, namespaceAliases: mergeNamespaceAliases(merged?.namespaceAliases, input.namespaceAliases), mcpServers: { ...merged?.mcpServers, @@ -2550,7 +2560,9 @@ function mergeConfigInputsWithSources(...inputs: Array void>(); private lastExposureSnapshot: ExposureSnapshot | undefined; private watchers: FSWatcher[] = []; @@ -104,6 +127,17 @@ export class CapletsEngine { options.configLoader ?? runtimeConfigLoader(options.authDir, options.vaultRecoveryTarget); const config = this.loadConfigWithWarnings(); this.registry = new ServerRegistry(config); + this.telemetry = createRuntimeTelemetryContext({ + config: this.registry.config, + env: options.telemetryEnv, + stateDir: options.telemetryStateDir, + surface: options.telemetrySurface ?? "serve", + visibility: options.telemetryVisibility ?? "unknown", + runtimeMode: options.telemetryRuntimeMode ?? runtimeModeFromEnv(options.telemetryEnv), + integration: options.telemetryIntegration, + debugSink: options.telemetryDebugSink, + dispatcher: options.telemetryDispatcher, + }); this.downstream = new DownstreamManager(this.registry, selectAuthOptions(options.authDir)); this.openapi = new OpenApiManager(this.registry, selectHttpLikeOptions(options)); this.googleDiscovery = new GoogleDiscoveryManager( @@ -199,9 +233,11 @@ export class CapletsEngine { } async execute(serverId: string, request: unknown): Promise { + const started = Date.now(); + let caplet: CapletConfig | undefined; try { - const caplet = this.registry.require(serverId); - return await handleServerTool( + caplet = this.registry.require(serverId); + const result = await handleServerTool( caplet, request, this.registry, @@ -218,8 +254,24 @@ export class CapletsEngine { }, this.googleDiscovery, ); + this.captureToolActivation( + caplet, + operationFromRequest(request), + "progressive", + result, + started, + ); + return result; } catch (error) { - return errorResult(error); + const result = errorResult(error); + this.captureToolActivation( + caplet, + operationFromRequest(request), + "progressive", + result, + started, + ); + return result; } } @@ -228,23 +280,35 @@ export class CapletsEngine { toolName: string, args: Record, ): Promise { + const started = Date.now(); + let caplet: CapletConfig | undefined; try { - const caplet = this.registry.require(serverId); + caplet = this.registry.require(serverId); const result = await this.callTool(caplet, toolName, args); - return annotateDirectResult(result, caplet, toolName); + const annotated = annotateDirectResult(result, caplet, toolName); + this.captureToolActivation(caplet, "call_tool", "direct", annotated, started); + return annotated; } catch (error) { - return errorResult(error); + const result = errorResult(error); + this.captureToolActivation(caplet, "call_tool", "direct", result, started); + return result; } } async readDirectResource(serverId: string, downstreamUri: string): Promise { + const started = Date.now(); + let caplet: CapletConfig | undefined; try { - const caplet = this.registry.require(serverId); + caplet = this.registry.require(serverId); if (caplet.backend !== "mcp") throw new Error(`Caplet ${serverId} has no MCP resources`); const result = await this.downstream.readResource(caplet, downstreamUri); - return annotateDirectResult(result, caplet, "read_resource"); + const annotated = annotateDirectResult(result, caplet, "read_resource"); + this.captureToolActivation(caplet, "read_resource", "direct", annotated, started); + return annotated; } catch (error) { - return errorResult(error); + const result = errorResult(error); + this.captureToolActivation(caplet, "read_resource", "direct", result, started); + return result; } } @@ -253,13 +317,19 @@ export class CapletsEngine { promptName: string, args: Record, ): Promise { + const started = Date.now(); + let caplet: CapletConfig | undefined; try { - const caplet = this.registry.require(serverId); + caplet = this.registry.require(serverId); if (caplet.backend !== "mcp") throw new Error(`Caplet ${serverId} has no MCP prompts`); const result = await this.downstream.getPrompt(caplet, promptName, args); - return annotateDirectResult(result, caplet, promptName); + const annotated = annotateDirectResult(result, caplet, promptName); + this.captureToolActivation(caplet, "get_prompt", "direct", annotated, started); + return annotated; } catch (error) { - return errorResult(error); + const result = errorResult(error); + this.captureToolActivation(caplet, "get_prompt", "direct", result, started); + return result; } } @@ -296,6 +366,17 @@ export class CapletsEngine { }); } + async captureCodeModeOutcome( + envelope: unknown, + options: { started: number; timeoutMs?: number | undefined }, + ): Promise { + await captureRuntimeTelemetryEvent( + this.telemetry, + "caplets_code_mode_outcome", + codeModeTelemetryProperties(envelope, Date.now() - options.started, options.timeoutMs), + ); + } + async close(): Promise { this.closed = true; try { @@ -314,6 +395,7 @@ export class CapletsEngine { this.closeWatchers(); await this.downstream.close(); await this.capletSets.close(); + await this.telemetry.dispatcher.shutdown(); this.reloadListeners.clear(); } } @@ -402,6 +484,7 @@ export class CapletsEngine { const previousConfig = this.registry.config; const nextRegistry = new ServerRegistry(nextConfig); this.registry = nextRegistry; + this.telemetry.config = nextConfig; this.downstream.updateRegistry(nextRegistry); this.openapi.updateRegistry(nextRegistry); this.googleDiscovery.updateRegistry(nextRegistry); @@ -578,6 +661,26 @@ export class CapletsEngine { } }, this.watchDebounceMs); } + + private captureToolActivation( + caplet: CapletConfig | undefined, + operation: unknown, + exposureMode: "direct" | "progressive" | "code_mode" | "mixed" | "unknown", + result: unknown, + started: number, + ): void { + void captureRuntimeTelemetryEvent(this.telemetry, "caplets_tool_activation", { + command_family: commandFamilyForTelemetrySurface(this.telemetry.surface), + ...toolActivationProperties({ + config: this.registry.config, + caplet, + operation, + exposureMode, + result, + durationMs: Date.now() - started, + }), + }).catch(() => undefined); + } } function runtimeConfigLoader( @@ -719,3 +822,22 @@ function isUnsupportedCapability(error: unknown): boolean { function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } + +function operationFromRequest(request: unknown): unknown { + return isRecord(request) ? request.operation : undefined; +} + +function runtimeModeFromEnv(env: NodeJS.ProcessEnv | undefined): RuntimeMode { + const mode = env?.CAPLETS_MODE ?? process.env.CAPLETS_MODE; + if (mode === "local" || mode === "remote" || mode === "cloud") return mode; + return "unknown"; +} + +function commandFamilyForTelemetrySurface(surface: TelemetrySurface) { + if (surface === "serve") return "serve"; + if (surface === "attach") return "attach"; + if (surface === "daemon") return "daemon"; + if (surface === "code_mode") return "code_mode"; + if (surface === "native") return "native"; + return "tools"; +} diff --git a/packages/core/src/native/service.ts b/packages/core/src/native/service.ts index 20018ea0..25bd25c0 100644 --- a/packages/core/src/native/service.ts +++ b/packages/core/src/native/service.ts @@ -19,6 +19,13 @@ import { } from "./remote"; import { CapletsEngine } from "../engine"; import { CapletsError } from "../errors"; +import type { + RuntimeMode, + TelemetryDebugSink, + TelemetryDispatcher, + TelemetrySurface, + TelemetryVisibility, +} from "../telemetry"; import { nativeCapletPromptGuidance, nativeCapletToolDescription, @@ -77,6 +84,14 @@ export type NativeCapletsServiceOptions = NativeCapletsServiceResolutionInput & writeErr?: (value: string) => void; remoteClientFactory?: (options: ResolvedNativeRemoteOptions) => RemoteCapletsClient; localServiceFactory?: (options: LocalNativeCapletsServiceOptions) => NativeCapletsService; + telemetryStateDir?: string | undefined; + telemetryEnv?: NodeJS.ProcessEnv | undefined; + telemetrySurface?: TelemetrySurface | undefined; + telemetryVisibility?: TelemetryVisibility | undefined; + telemetryRuntimeMode?: RuntimeMode | undefined; + telemetryIntegration?: "opencode" | "pi" | "native" | "unknown" | undefined; + telemetryDebugSink?: TelemetryDebugSink | undefined; + telemetryDispatcher?: TelemetryDispatcher | undefined; }; export type NativeCapletTool = { @@ -102,6 +117,10 @@ export type NativeCapletsToolsChangedListener = (tools: NativeCapletTool[]) => v export type NativeCapletsService = { listTools(): NativeCapletTool[]; execute(capletId: string, request: unknown): Promise; + captureCodeModeOutcome?( + envelope: unknown, + options: { started: number; timeoutMs?: number | undefined }, + ): Promise; codeModeService?(): NativeCapletsService; reload(): Promise; onToolsChanged(listener: NativeCapletsToolsChangedListener): () => void; @@ -163,6 +182,10 @@ class DefaultNativeCapletsService implements NativeCapletsService { this.engine = new CapletsEngine({ ...options, writeErr: this.writeErr, + telemetrySurface: options.telemetrySurface ?? "native", + telemetryVisibility: options.telemetryVisibility ?? "hidden", + telemetryRuntimeMode: options.telemetryRuntimeMode ?? runtimeModeFromNativeOptions(options), + telemetryIntegration: options.telemetryIntegration ?? "native", }); this.postReloadRefresh = this.refreshExposureSnapshot({ emitToolsChanged: this.hasSnapshotBackedDirectExposure(), @@ -205,11 +228,22 @@ class DefaultNativeCapletsService implements NativeCapletsService { async execute(capletId: string, request: unknown): Promise { if (capletId === nativeCodeModeToolId) { - return await executeCodeModeRunNative( + const started = Date.now(); + const envelope = await executeCodeModeRunNative( this.codeModeDelegate(), request, this.codeModeSessions, ); + const parsed = codeModeRunInputSchema.safeParse(request); + void this.engine + .captureCodeModeOutcome(envelope, { + started, + ...(parsed.success && parsed.data.timeoutMs !== undefined + ? { timeoutMs: parsed.data.timeoutMs } + : {}), + }) + .catch(() => undefined); + return envelope; } const route = this.directToolRoutes.get(capletId); if (route) { @@ -228,6 +262,13 @@ class DefaultNativeCapletsService implements NativeCapletsService { return await this.engine.execute(capletId, request); } + async captureCodeModeOutcome( + envelope: unknown, + options: { started: number; timeoutMs?: number | undefined }, + ): Promise { + await this.engine.captureCodeModeOutcome(envelope, options); + } + codeModeService(): NativeCapletsService { return this.codeModeDelegate(); } @@ -525,6 +566,15 @@ function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } +function runtimeModeFromNativeOptions(options: NativeCapletsServiceOptions) { + if (options.mode === "remote") return "remote"; + if (options.mode === "cloud") return "cloud"; + if (options.remote?.url) return "remote"; + const envMode = options.telemetryEnv?.CAPLETS_MODE ?? process.env.CAPLETS_MODE; + if (envMode === "remote" || envMode === "cloud" || envMode === "local") return envMode; + return "local"; +} + function codeModeRunNativeTool(capletTools: NativeCapletTool[]): NativeCapletTool { const codeModeCaplets = capletTools.map((tool) => ({ id: tool.caplet, diff --git a/packages/core/src/serve/session.ts b/packages/core/src/serve/session.ts index ba21afe5..4fca276c 100644 --- a/packages/core/src/serve/session.ts +++ b/packages/core/src/serve/session.ts @@ -224,6 +224,7 @@ export class CapletsMcpSession { } private async handleCodeModeRunTool(request: unknown): Promise { + const started = Date.now(); const parsed = codeModeRunInputSchema.safeParse(request); const envelope = parsed.success ? await runCodeMode({ @@ -247,6 +248,14 @@ export class CapletsMcpSession { logs: { entries: [], truncated: false, stored: false }, meta: emptyCodeModeRunMeta(), }; + void this.engine + .captureCodeModeOutcome(envelope, { + started, + ...(parsed.success && parsed.data.timeoutMs !== undefined + ? { timeoutMs: parsed.data.timeoutMs } + : {}), + }) + .catch(() => undefined); return { content: [{ type: "text" as const, text: JSON.stringify(envelope, null, 2) }], structuredContent: envelope, diff --git a/packages/core/src/telemetry/context.ts b/packages/core/src/telemetry/context.ts new file mode 100644 index 00000000..d5f30f15 --- /dev/null +++ b/packages/core/src/telemetry/context.ts @@ -0,0 +1,80 @@ +import type { CapletsConfig } from "../config"; +import { + readTelemetryIdentity, + readTelemetryNotice, + type TelemetryExecutionContext, + type TelemetryState, + type TelemetryStateOptions, + type TelemetrySurface, + type TelemetryVisibility, +} from "./state"; + +export type ResolveTelemetryStateOptions = TelemetryStateOptions & { + config?: Pick | undefined; + env?: NodeJS.ProcessEnv | undefined; + surface: TelemetrySurface; + visibility: TelemetryVisibility; + debug?: boolean | undefined; + allowWithoutNotice?: boolean | undefined; + createIdentity?: boolean | undefined; +}; + +export function resolveTelemetryState(options: ResolveTelemetryStateOptions): TelemetryState { + const env = options.env ?? process.env; + const executionContext = classifyExecutionContext(env); + const notice = readTelemetryNotice(options); + const base = { + surface: options.surface, + visibility: options.visibility, + executionContext, + notice, + stateDir: options.stateDir, + }; + + if (options.debug) { + return { ...base, status: "debug", decider: "debug" }; + } + if (env.CAPLETS_DISABLE_TELEMETRY === "1") { + return { ...base, status: "disabled", decider: "env" }; + } + if (options.config?.telemetry === false) { + return { ...base, status: "disabled", decider: "config" }; + } + if (isTestEnv(env)) { + return { ...base, status: "disabled", decider: "test" }; + } + + if ( + executionContext !== "ci" && + !notice.shown && + !options.allowWithoutNotice && + options.visibility !== "visible" + ) { + return { ...base, status: "suppressed", decider: "notice" }; + } + + return { + ...base, + status: "enabled", + decider: "default", + identity: readTelemetryIdentity({ ...options, create: options.createIdentity !== false }), + }; +} + +export function classifyExecutionContext( + env: NodeJS.ProcessEnv = process.env, +): TelemetryExecutionContext { + if (env.CI === "true" || env.GITHUB_ACTIONS === "true" || env.BUILDKITE === "true") { + return "ci"; + } + return process.stdout.isTTY || process.stderr.isTTY ? "interactive" : "noninteractive"; +} + +function isTestEnv(env: NodeJS.ProcessEnv): boolean { + return ( + env.NODE_ENV === "test" || + env.VITEST === "true" || + env.VITEST_WORKER_ID !== undefined || + env.CAPLETS_TEST === "1" + ); +} diff --git a/packages/core/src/telemetry/debug.ts b/packages/core/src/telemetry/debug.ts new file mode 100644 index 00000000..fe9245f1 --- /dev/null +++ b/packages/core/src/telemetry/debug.ts @@ -0,0 +1,18 @@ +import type { TelemetryEvent } from "./events"; + +export type TelemetryDebugRecord = { + state: "debug" | "enabled" | "disabled" | "suppressed"; + event: TelemetryEvent; +}; + +export class TelemetryDebugSink { + readonly records: TelemetryDebugRecord[] = []; + + capture(state: TelemetryDebugRecord["state"], event: TelemetryEvent): void { + this.records.push({ state, event }); + } + + toJSON(): TelemetryDebugRecord[] { + return this.records; + } +} diff --git a/packages/core/src/telemetry/delivery.ts b/packages/core/src/telemetry/delivery.ts new file mode 100644 index 00000000..1811d898 --- /dev/null +++ b/packages/core/src/telemetry/delivery.ts @@ -0,0 +1,5 @@ +export { + readTelemetryDeliveryHealth, + recordTelemetryDrop, + type TelemetryDeliveryHealth, +} from "./state"; diff --git a/packages/core/src/telemetry/events.ts b/packages/core/src/telemetry/events.ts new file mode 100644 index 00000000..beb178d1 --- /dev/null +++ b/packages/core/src/telemetry/events.ts @@ -0,0 +1,175 @@ +import type { TelemetryExecutionContext, TelemetrySurface } from "./state"; +import { assertTelemetrySafeProperties } from "./privacy"; + +export type RuntimeMode = "local" | "remote" | "cloud" | "unknown"; +export type CommandFamily = + | "init" + | "setup" + | "add" + | "install" + | "auth" + | "remote" + | "doctor" + | "serve" + | "attach" + | "daemon" + | "inspect" + | "check" + | "tools" + | "resources" + | "prompts" + | "complete" + | "code_mode" + | "native" + | "telemetry" + | "unknown"; +export type Outcome = "success" | "failure" | "cancelled" | "timeout" | "suppressed"; +export type DurationBucket = "lt_100ms" | "lt_1s" | "lt_5s" | "lt_30s" | "gte_30s"; +export type TimeoutBucket = "none" | "lt_1s" | "lt_10s" | "lt_60s" | "gte_60s"; +export type DiagnosticCategory = + | "config" + | "auth" + | "network" + | "runtime" + | "validation" + | "code_mode" + | "provider" + | "unknown"; + +export type TelemetryProductEventName = + | "caplets_cli_command" + | "caplets_setup_milestone" + | "caplets_runtime_lifecycle" + | "caplets_tool_activation" + | "caplets_code_mode_outcome" + | "caplets_delivery_health"; + +export type TelemetryReliabilityEventName = "caplets_reliability_error"; + +export type TelemetryPropertyValue = string | number | boolean; + +export type TelemetryProperties = Partial<{ + package: string; + version: string; + surface: TelemetrySurface; + runtime_mode: RuntimeMode; + execution_context: TelemetryExecutionContext; + command_family: CommandFamily; + operation_family: CommandFamily; + outcome: Outcome; + duration_bucket: DurationBucket; + timeout_bucket: TimeoutBucket; + integration: "opencode" | "pi" | "native" | "unknown"; + exposure_mode: "direct" | "progressive" | "code_mode" | "mixed" | "unknown"; + backend_mcp_count: number; + backend_openapi_count: number; + backend_google_discovery_count: number; + backend_graphql_count: number; + backend_http_count: number; + backend_cli_count: number; + backend_caplets_count: number; + direct_count: number; + progressive_count: number; + code_mode_count: number; + session_category: "created" | "reused" | "none" | "unknown"; + any_caplet_invoked: boolean; + provider: "posthog" | "sentry"; + reason: string; + count_bucket: string; + error_code: string; + diagnostic_category: DiagnosticCategory; + os_family: NodeJS.Platform | "unknown"; + arch: NodeJS.Architecture | "unknown"; + node_major: number; +}>; + +export type ProductTelemetryEvent = { + provider: "posthog"; + name: TelemetryProductEventName; + distinctId: string; + properties: TelemetryProperties & { $process_person_profile: false }; +}; + +export type ReliabilityTelemetryEvent = { + provider: "sentry"; + name: TelemetryReliabilityEventName; + tags: Record; + fingerprint: string[]; +}; + +export type TelemetryEvent = ProductTelemetryEvent | ReliabilityTelemetryEvent; + +const PRODUCT_EVENTS = new Set([ + "caplets_cli_command", + "caplets_setup_milestone", + "caplets_runtime_lifecycle", + "caplets_tool_activation", + "caplets_code_mode_outcome", + "caplets_delivery_health", +]); + +const RELIABILITY_EVENTS = new Set(["caplets_reliability_error"]); + +export function buildProductTelemetryEvent(input: { + name: TelemetryProductEventName; + distinctId: string; + properties: TelemetryProperties; +}): ProductTelemetryEvent { + if (!PRODUCT_EVENTS.has(input.name)) { + throw new Error(`unknown telemetry event: ${input.name}`); + } + assertTelemetrySafeProperties(input.properties); + return { + provider: "posthog", + name: input.name, + distinctId: input.distinctId, + properties: { + $process_person_profile: false, + ...input.properties, + }, + }; +} + +export function buildReliabilityTelemetryEvent(input: { + name: TelemetryReliabilityEventName; + properties: TelemetryProperties; +}): ReliabilityTelemetryEvent { + if (!RELIABILITY_EVENTS.has(input.name)) { + throw new Error(`unknown telemetry event: ${input.name}`); + } + assertTelemetrySafeProperties(input.properties); + const tags = tagsFor(input.properties); + return { + provider: "sentry", + name: input.name, + tags, + fingerprint: [ + tags.package ?? "unknown", + tags.surface ?? "unknown", + tags.command_family ?? "unknown", + tags.runtime_mode ?? "unknown", + tags.error_code ?? "unknown", + tags.diagnostic_category ?? "unknown", + ], + }; +} + +export function durationBucket(ms: number): DurationBucket { + if (ms < 100) return "lt_100ms"; + if (ms < 1_000) return "lt_1s"; + if (ms < 5_000) return "lt_5s"; + if (ms < 30_000) return "lt_30s"; + return "gte_30s"; +} + +export function timeoutBucket(ms: number | undefined): TimeoutBucket { + if (ms === undefined) return "none"; + if (ms < 1_000) return "lt_1s"; + if (ms < 10_000) return "lt_10s"; + if (ms < 60_000) return "lt_60s"; + return "gte_60s"; +} + +function tagsFor(properties: TelemetryProperties): Record { + return Object.fromEntries(Object.entries(properties).map(([key, value]) => [key, String(value)])); +} diff --git a/packages/core/src/telemetry/identity.ts b/packages/core/src/telemetry/identity.ts new file mode 100644 index 00000000..2d747890 --- /dev/null +++ b/packages/core/src/telemetry/identity.ts @@ -0,0 +1,6 @@ +export { + deleteTelemetryIdentity, + readTelemetryIdentity, + rotateTelemetryIdentity, + type TelemetryIdentity, +} from "./state"; diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts new file mode 100644 index 00000000..40265eea --- /dev/null +++ b/packages/core/src/telemetry/index.ts @@ -0,0 +1,64 @@ +export { + classifyExecutionContext, + resolveTelemetryState, + type ResolveTelemetryStateOptions, +} from "./context"; +export { + deleteTelemetryIdentity, + readTelemetryDeliveryHealth, + readTelemetryIdentity, + readTelemetryNotice, + recordTelemetryDrop, + recordTelemetryNoticeShown, + rotateTelemetryIdentity, + telemetryDeliveryHealthPath, + telemetryIdentityPath, + telemetryNoticePath, + telemetryStateDir, + type TelemetryDeliveryHealth, + type TelemetryExecutionContext, + type TelemetryIdentity, + type TelemetryNoticeState, + type TelemetryState, + type TelemetryStateDecider, + type TelemetryStateOptions, + type TelemetryStateStatus, + type TelemetrySurface, + type TelemetryVisibility, +} from "./state"; +export { maybePrintTelemetryNotice, TELEMETRY_NOTICE } from "./notice"; +export { + buildProductTelemetryEvent, + buildReliabilityTelemetryEvent, + durationBucket, + timeoutBucket, + type CommandFamily, + type DiagnosticCategory, + type RuntimeMode, + type ProductTelemetryEvent, + type ReliabilityTelemetryEvent, + type TelemetryEvent, + type TelemetryProductEventName, + type TelemetryProperties, + type TelemetryReliabilityEventName, +} from "./events"; +export { assertTelemetrySafeProperties, stripSentryEvent } from "./privacy"; +export { TelemetryDebugSink, type TelemetryDebugRecord } from "./debug"; +export { + createTelemetryDispatcher, + type TelemetryDispatcher, + type TelemetryDispatcherOptions, + type TelemetryProviderFactories, +} from "./providers"; +export { + backendFamilyCounts, + captureRuntimeTelemetryEvent, + codeModeTelemetryProperties, + createRuntimeTelemetryContext, + exposureModeCounts, + operationFamilyFromOperation, + outcomeFromResult, + toolActivationProperties, + type RuntimeTelemetryContext, + type RuntimeTelemetryOptions, +} from "./runtime"; diff --git a/packages/core/src/telemetry/notice.ts b/packages/core/src/telemetry/notice.ts new file mode 100644 index 00000000..5ef40ca9 --- /dev/null +++ b/packages/core/src/telemetry/notice.ts @@ -0,0 +1,21 @@ +import { + recordTelemetryNoticeShown, + type TelemetrySurface, + type TelemetryStateOptions, +} from "./state"; + +export const TELEMETRY_NOTICE = + "Caplets collects anonymous telemetry for product usage and reliability. Disable it with CAPLETS_DISABLE_TELEMETRY=1 or `caplets telemetry disable`.\n"; + +export type TelemetryNoticeOptions = TelemetryStateOptions & { + surface: TelemetrySurface; + stderrIsTTY?: boolean | undefined; + writeErr: (text: string) => void; +}; + +export function maybePrintTelemetryNotice(options: TelemetryNoticeOptions): boolean { + if (!options.stderrIsTTY) return false; + options.writeErr(TELEMETRY_NOTICE); + recordTelemetryNoticeShown({ ...options, surface: options.surface }); + return true; +} diff --git a/packages/core/src/telemetry/privacy.ts b/packages/core/src/telemetry/privacy.ts new file mode 100644 index 00000000..590c30c3 --- /dev/null +++ b/packages/core/src/telemetry/privacy.ts @@ -0,0 +1,152 @@ +import type { TelemetryProperties } from "./events"; + +const ALLOWED_PROPERTY_KEYS = new Set([ + "$process_person_profile", + "$geoip_disable", + "package", + "version", + "surface", + "runtime_mode", + "execution_context", + "command_family", + "operation_family", + "outcome", + "duration_bucket", + "timeout_bucket", + "integration", + "exposure_mode", + "backend_mcp_count", + "backend_openapi_count", + "backend_google_discovery_count", + "backend_graphql_count", + "backend_http_count", + "backend_cli_count", + "backend_caplets_count", + "direct_count", + "progressive_count", + "code_mode_count", + "session_category", + "any_caplet_invoked", + "provider", + "reason", + "count_bucket", + "error_code", + "diagnostic_category", + "os_family", + "arch", + "node_major", +]); + +const SAFE_STRING = /^[a-zA-Z0-9@._:-]{1,80}$/u; +const SAFE_PACKAGE = /^@?[a-zA-Z0-9._-]+(?:\/[a-zA-Z0-9._-]+)?$/u; +const SUSPICIOUS_VALUE = [ + /^\/|^[A-Za-z]:\\/u, + /^https?:\/\//iu, + /[a-z0-9-]+\.[a-z]{2,}/iu, + /(?:^|[_.-])token(?:$|[_.=-])/iu, + /(?:^|[_.-])secret(?:$|[_.=-])/iu, + /(?:^|[_.-])key(?:$|[_.=-])/iu, + /^sk-[a-z0-9]/iu, + /^gh[pousr]_[a-z0-9]/iu, + /^[A-Z_]{3,}=.+/u, +]; +const COMMAND_FAMILIES = new Set([ + "init", + "setup", + "add", + "install", + "auth", + "remote", + "doctor", + "serve", + "attach", + "daemon", + "inspect", + "check", + "tools", + "resources", + "prompts", + "complete", + "code_mode", + "native", + "telemetry", + "unknown", +]); +const STRING_VALUE_ALLOWLISTS: Record> = { + surface: new Set(["cli", "serve", "attach", "daemon", "native", "code_mode"]), + runtime_mode: new Set(["local", "remote", "cloud", "unknown"]), + execution_context: new Set(["interactive", "noninteractive", "ci"]), + command_family: COMMAND_FAMILIES, + operation_family: COMMAND_FAMILIES, + outcome: new Set(["success", "failure", "cancelled", "timeout", "suppressed"]), + duration_bucket: new Set(["lt_100ms", "lt_1s", "lt_5s", "lt_30s", "gte_30s"]), + timeout_bucket: new Set(["none", "lt_1s", "lt_10s", "lt_60s", "gte_60s"]), + integration: new Set(["opencode", "pi", "native", "unknown"]), + exposure_mode: new Set(["direct", "progressive", "code_mode", "mixed", "unknown"]), + session_category: new Set(["created", "reused", "none", "unknown"]), + provider: new Set(["posthog", "sentry"]), + reason: new Set(["not_configured", "send_failed"]), + diagnostic_category: new Set([ + "config", + "auth", + "network", + "runtime", + "validation", + "code_mode", + "provider", + "unknown", + ]), +}; + +export function assertTelemetrySafeProperties( + properties: TelemetryProperties & Record, +): void { + for (const [key, value] of Object.entries(properties)) { + if (!ALLOWED_PROPERTY_KEYS.has(key)) { + throw new Error(`unknown telemetry property: ${key}`); + } + if (typeof value === "string") { + const valueAllowlist = STRING_VALUE_ALLOWLISTS[key]; + if (valueAllowlist && !valueAllowlist.has(value)) { + throw new Error(`unsafe telemetry property ${key}`); + } + if (key === "error_code" && !/^[A-Z_]{2,80}$/u.test(value)) { + throw new Error(`unsafe telemetry property ${key}`); + } + const safeShape = key === "package" ? SAFE_PACKAGE.test(value) : SAFE_STRING.test(value); + if ( + !safeShape || + (key !== "package" && + key !== "version" && + SUSPICIOUS_VALUE.some((pattern) => pattern.test(value))) + ) { + throw new Error(`unsafe telemetry property ${key}`); + } + continue; + } + if (typeof value === "number") { + if (!Number.isFinite(value) || value < 0 || !Number.isInteger(value)) { + throw new Error(`unsafe telemetry property ${key}`); + } + continue; + } + if (typeof value === "boolean") { + continue; + } + throw new Error(`unsafe telemetry property ${key}`); + } +} + +export function stripSentryEvent(event: Record): Record { + const stripped: Record = {}; + if (event.tags && typeof event.tags === "object" && !Array.isArray(event.tags)) { + stripped.tags = event.tags; + } + if (Array.isArray(event.fingerprint)) { + stripped.fingerprint = event.fingerprint; + } + if (event.level) { + stripped.level = event.level; + } + return stripped; +} diff --git a/packages/core/src/telemetry/providers.ts b/packages/core/src/telemetry/providers.ts new file mode 100644 index 00000000..8398b08f --- /dev/null +++ b/packages/core/src/telemetry/providers.ts @@ -0,0 +1,141 @@ +import type { NodeClient, NodeOptions } from "@sentry/node"; +import type { PostHog } from "posthog-node"; +import type { ProductTelemetryEvent, ReliabilityTelemetryEvent, TelemetryEvent } from "./events"; +import { stripSentryEvent } from "./privacy"; +import { recordTelemetryDrop, type TelemetryState } from "./state"; + +export type PostHogClient = Pick; +export type SentryClient = Pick; + +export type TelemetryProviderFactories = { + createPostHog?: ((token: string) => Promise | PostHogClient) | undefined; + createSentry?: ((dsn: string) => Promise | SentryClient) | undefined; +}; + +export type TelemetryDispatcherOptions = { + posthogToken?: string | undefined; + sentryDsn?: string | undefined; + stateDir?: string | undefined; + factories?: TelemetryProviderFactories | undefined; +}; + +export type TelemetryDispatcher = { + capture(state: TelemetryState, event: TelemetryEvent): Promise; + shutdown(): Promise; +}; + +export function createTelemetryDispatcher( + options: TelemetryDispatcherOptions = {}, +): TelemetryDispatcher { + let posthog: Promise | undefined; + let sentry: Promise | undefined; + + async function posthogClient(): Promise { + const token = options.posthogToken ?? process.env.CAPLETS_POSTHOG_TOKEN; + if (!token) return undefined; + posthog ??= Promise.resolve((options.factories?.createPostHog ?? defaultPostHogFactory)(token)); + return posthog; + } + + async function sentryClient(): Promise { + const dsn = options.sentryDsn ?? process.env.CAPLETS_SENTRY_DSN; + if (!dsn) return undefined; + sentry ??= Promise.resolve((options.factories?.createSentry ?? defaultSentryFactory)(dsn)); + return sentry; + } + + return { + async capture(state, event) { + if (state.status !== "enabled") { + return; + } + try { + if (event.provider === "posthog") { + await capturePostHog(await posthogClient(), event, state.stateDir ?? options.stateDir); + return; + } + await captureSentry(await sentryClient(), event, state.stateDir ?? options.stateDir); + } catch { + recordTelemetryDrop({ + stateDir: state.stateDir ?? options.stateDir, + provider: event.provider, + reason: "send_failed", + }); + } + }, + async shutdown() { + const clients = await Promise.allSettled([posthog, sentry]); + for (const client of clients) { + if (client.status !== "fulfilled" || !client.value) continue; + if ("shutdown" in client.value) { + await Promise.resolve(client.value.shutdown()).catch(() => undefined); + } + if ("flush" in client.value) { + await Promise.resolve(client.value.flush(2_000)).catch(() => undefined); + } + } + }, + }; +} + +async function capturePostHog( + client: PostHogClient | undefined, + event: ProductTelemetryEvent, + stateDir: string | undefined, +): Promise { + if (!client) { + recordTelemetryDrop({ stateDir, provider: "posthog", reason: "not_configured" }); + return; + } + client.capture({ + distinctId: event.distinctId, + event: event.name, + properties: { + $geoip_disable: true, + ...event.properties, + }, + }); +} + +async function captureSentry( + client: SentryClient | undefined, + event: ReliabilityTelemetryEvent, + stateDir: string | undefined, +): Promise { + if (!client) { + recordTelemetryDrop({ stateDir, provider: "sentry", reason: "not_configured" }); + return; + } + client.captureEvent({ + level: "error", + tags: event.tags, + fingerprint: event.fingerprint, + }); +} + +async function defaultPostHogFactory(token: string): Promise { + const { PostHog } = await import("posthog-node"); + return new PostHog(token, { + flushAt: 20, + flushInterval: 10_000, + disableGeoip: true, + historicalMigration: false, + }); +} + +async function defaultSentryFactory(dsn: string): Promise { + const sentry = await import("@sentry/node"); + const options = { + dsn, + sendDefaultPii: false, + defaultIntegrations: false, + tracesSampleRate: 0, + beforeSend(event) { + return stripSentryEvent( + event as unknown as Record, + ) as unknown as typeof event; + }, + } satisfies NodeOptions; + sentry.init(options); + return sentry.getClient() as NodeClient; +} diff --git a/packages/core/src/telemetry/runtime.ts b/packages/core/src/telemetry/runtime.ts new file mode 100644 index 00000000..8604c79f --- /dev/null +++ b/packages/core/src/telemetry/runtime.ts @@ -0,0 +1,205 @@ +import { version as packageJsonVersion } from "../../package.json"; +import type { CapletConfig, CapletsConfig } from "../config"; +import { resolveExposure } from "../exposure/policy"; +import { + buildProductTelemetryEvent, + durationBucket, + timeoutBucket, + type CommandFamily, + type Outcome, + type RuntimeMode, + type TelemetryProductEventName, + type TelemetryProperties, +} from "./events"; +import { TelemetryDebugSink } from "./debug"; +import { createTelemetryDispatcher, type TelemetryDispatcher } from "./providers"; +import { resolveTelemetryState } from "./context"; +import { readTelemetryIdentity, type TelemetrySurface, type TelemetryVisibility } from "./state"; + +export type RuntimeTelemetryOptions = { + config: CapletsConfig; + env?: NodeJS.ProcessEnv | undefined; + stateDir?: string | undefined; + surface: TelemetrySurface; + visibility: TelemetryVisibility; + runtimeMode?: RuntimeMode | undefined; + integration?: "opencode" | "pi" | "native" | "unknown" | undefined; + debugSink?: TelemetryDebugSink | undefined; + dispatcher?: TelemetryDispatcher | undefined; +}; + +export type RuntimeTelemetryContext = RuntimeTelemetryOptions & { + dispatcher: TelemetryDispatcher; +}; + +export function createRuntimeTelemetryContext( + options: RuntimeTelemetryOptions, +): RuntimeTelemetryContext { + return { + ...options, + dispatcher: options.dispatcher ?? createTelemetryDispatcher({ stateDir: options.stateDir }), + }; +} + +export async function captureRuntimeTelemetryEvent( + context: RuntimeTelemetryContext, + name: TelemetryProductEventName, + properties: TelemetryProperties, +): Promise { + const state = resolveTelemetryState({ + config: context.config, + env: context.env, + stateDir: context.stateDir, + surface: context.surface, + visibility: context.visibility, + debug: context.debugSink !== undefined, + }); + if (state.status !== "enabled" && state.status !== "debug") { + return; + } + const identity = + state.identity ?? readTelemetryIdentity({ stateDir: context.stateDir, create: false }); + const event = buildProductTelemetryEvent({ + name, + distinctId: identity.id, + properties: { + package: "@caplets/core", + version: packageJsonVersion, + surface: context.surface, + runtime_mode: context.runtimeMode ?? "unknown", + execution_context: state.executionContext, + ...(context.integration ? { integration: context.integration } : {}), + ...properties, + }, + }); + if (state.status === "debug") { + context.debugSink?.capture("debug", event); + return; + } + await context.dispatcher.capture(state, event); +} + +export function backendFamilyCounts(config: CapletsConfig): TelemetryProperties { + return { + backend_mcp_count: enabledCount(config.mcpServers), + backend_openapi_count: enabledCount(config.openapiEndpoints), + backend_google_discovery_count: enabledCount(config.googleDiscoveryApis), + backend_graphql_count: enabledCount(config.graphqlEndpoints), + backend_http_count: enabledCount(config.httpApis), + backend_cli_count: enabledCount(config.cliTools), + backend_caplets_count: enabledCount(config.capletSets), + }; +} + +export function exposureModeCounts(config: CapletsConfig): TelemetryProperties { + let direct = 0; + let progressive = 0; + let codeMode = 0; + for (const caplet of allCaplets(config)) { + if (caplet.disabled || caplet.setup || caplet.projectBinding?.required) continue; + const exposure = resolveExposure(caplet.exposure, config.options.exposure); + if (exposure.direct) direct += 1; + if (exposure.progressive) progressive += 1; + if (exposure.codeMode) codeMode += 1; + } + return { + direct_count: direct, + progressive_count: progressive, + code_mode_count: codeMode, + }; +} + +export function operationFamilyFromOperation(operation: unknown): CommandFamily { + if (operation === "inspect") return "inspect"; + if (operation === "check") return "check"; + if ( + operation === "tools" || + operation === "search_tools" || + operation === "get_tool" || + operation === "call_tool" + ) { + return "tools"; + } + if ( + operation === "resources" || + operation === "resource_templates" || + operation === "read_resource" || + operation === "list_resources" || + operation === "list_resource_templates" + ) { + return "resources"; + } + if (operation === "prompts" || operation === "get_prompt" || operation === "list_prompts") { + return "prompts"; + } + if (operation === "complete") return "complete"; + if (operation === "code_mode") return "code_mode"; + return "unknown"; +} + +export function outcomeFromResult(result: unknown): Outcome { + if (isRecord(result) && result.isError === true) return "failure"; + if (isRecord(result) && result.ok === false) { + const code = isRecord(result.error) ? result.error.code : undefined; + if (code === "TIMEOUT") return "timeout"; + return "failure"; + } + return "success"; +} + +export function codeModeTelemetryProperties( + envelope: unknown, + durationMs: number, + timeoutMs: number | undefined, +): TelemetryProperties { + const record = isRecord(envelope) ? envelope : {}; + const meta = isRecord(record.meta) ? record.meta : {}; + const sessionStatus = meta.sessionStatus; + return { + command_family: "code_mode", + outcome: outcomeFromResult(record), + duration_bucket: durationBucket(durationMs), + timeout_bucket: timeoutBucket(timeoutMs), + session_category: + sessionStatus === "created" || sessionStatus === "reused" ? sessionStatus : "unknown", + any_caplet_invoked: false, + }; +} + +export function toolActivationProperties(input: { + config: CapletsConfig; + caplet: CapletConfig | undefined; + operation: unknown; + exposureMode: "direct" | "progressive" | "code_mode" | "mixed" | "unknown"; + result: unknown; + durationMs: number; +}): TelemetryProperties { + return { + operation_family: operationFamilyFromOperation(input.operation), + exposure_mode: input.exposureMode, + outcome: outcomeFromResult(input.result), + duration_bucket: durationBucket(input.durationMs), + ...backendFamilyCounts(input.config), + ...exposureModeCounts(input.config), + }; +} + +function enabledCount(record: Record): number { + return Object.values(record).filter((caplet) => !caplet.disabled).length; +} + +function allCaplets(config: CapletsConfig): CapletConfig[] { + return [ + ...Object.values(config.mcpServers), + ...Object.values(config.openapiEndpoints), + ...Object.values(config.googleDiscoveryApis), + ...Object.values(config.graphqlEndpoints), + ...Object.values(config.httpApis), + ...Object.values(config.cliTools), + ...Object.values(config.capletSets), + ]; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} diff --git a/packages/core/src/telemetry/state.ts b/packages/core/src/telemetry/state.ts new file mode 100644 index 00000000..4c7983ab --- /dev/null +++ b/packages/core/src/telemetry/state.ts @@ -0,0 +1,181 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { DEFAULT_TELEMETRY_STATE_DIR } from "../config/paths"; + +export type TelemetrySurface = "cli" | "serve" | "attach" | "daemon" | "native" | "code_mode"; +export type TelemetryVisibility = "visible" | "hidden" | "unknown"; +export type TelemetryExecutionContext = "interactive" | "noninteractive" | "ci"; +export type TelemetryStateStatus = "enabled" | "disabled" | "suppressed" | "debug"; +export type TelemetryStateDecider = "debug" | "env" | "config" | "test" | "notice" | "default"; + +export type TelemetryIdentity = + | { kind: "stable"; id: string } + | { kind: "ephemeral"; id: string; reason: "state-unavailable" }; + +export type TelemetryNoticeState = + | { shown: false } + | { shown: true; shownAt: string; surface: TelemetrySurface }; + +export type TelemetryDeliveryHealth = Record>; + +export type TelemetryState = { + status: TelemetryStateStatus; + decider: TelemetryStateDecider; + surface: TelemetrySurface; + visibility: TelemetryVisibility; + executionContext: TelemetryExecutionContext; + notice: TelemetryNoticeState; + stateDir?: string | undefined; + identity?: TelemetryIdentity | undefined; +}; + +export type TelemetryStateOptions = { + stateDir?: string | undefined; +}; + +export function telemetryStateDir(options: TelemetryStateOptions = {}): string { + return options.stateDir ?? DEFAULT_TELEMETRY_STATE_DIR; +} + +export function telemetryIdentityPath(options: TelemetryStateOptions = {}): string { + return join(telemetryStateDir(options), "identity.json"); +} + +export function telemetryNoticePath(options: TelemetryStateOptions = {}): string { + return join(telemetryStateDir(options), "notice.json"); +} + +export function telemetryDeliveryHealthPath(options: TelemetryStateOptions = {}): string { + return join(telemetryStateDir(options), "delivery-health.json"); +} + +export function readTelemetryIdentity( + options: TelemetryStateOptions & { create?: boolean | undefined } = {}, +): TelemetryIdentity { + const path = telemetryIdentityPath(options); + const existing = readJson<{ id?: unknown }>(path); + if (typeof existing?.id === "string" && /^anon_[a-f0-9]{32}$/u.test(existing.id)) { + return { kind: "stable", id: existing.id }; + } + + if (!options.create) { + return { kind: "ephemeral", id: ephemeralId(), reason: "state-unavailable" }; + } + + const identity = { id: stableId(), createdAt: new Date().toISOString() }; + if (!writePrivateJson(path, identity)) { + return { kind: "ephemeral", id: ephemeralId(), reason: "state-unavailable" }; + } + return { kind: "stable", id: identity.id }; +} + +export function rotateTelemetryIdentity(options: TelemetryStateOptions = {}): TelemetryIdentity { + const identity = { id: stableId(), createdAt: new Date().toISOString() }; + if (!writePrivateJson(telemetryIdentityPath(options), identity)) { + return { kind: "ephemeral", id: ephemeralId(), reason: "state-unavailable" }; + } + return { kind: "stable", id: identity.id }; +} + +export function deleteTelemetryIdentity(options: TelemetryStateOptions = {}): void { + rmSync(telemetryIdentityPath(options), { force: true }); +} + +export function readTelemetryNotice(options: TelemetryStateOptions = {}): TelemetryNoticeState { + const existing = readJson<{ + shown?: unknown; + shownAt?: unknown; + surface?: unknown; + }>(telemetryNoticePath(options)); + if ( + existing?.shown === true && + typeof existing.shownAt === "string" && + isTelemetrySurface(existing.surface) + ) { + return { shown: true, shownAt: existing.shownAt, surface: existing.surface }; + } + return { shown: false }; +} + +export function recordTelemetryNoticeShown( + options: TelemetryStateOptions & { surface: TelemetrySurface }, +): TelemetryNoticeState { + const notice = { + shown: true, + shownAt: new Date().toISOString(), + surface: options.surface, + } as const; + writePrivateJson(telemetryNoticePath(options), notice); + return notice; +} + +export function readTelemetryDeliveryHealth( + options: TelemetryStateOptions = {}, +): TelemetryDeliveryHealth { + const existing = readJson(telemetryDeliveryHealthPath(options)); + if (!isDeliveryHealth(existing)) { + return {}; + } + return existing; +} + +export function recordTelemetryDrop( + options: TelemetryStateOptions & { provider: string; reason: string }, +): void { + const health = readTelemetryDeliveryHealth(options); + health[options.provider] = health[options.provider] ?? {}; + health[options.provider]![options.reason] = (health[options.provider]![options.reason] ?? 0) + 1; + writePrivateJson(telemetryDeliveryHealthPath(options), health); +} + +export function writePrivateJson(path: string, value: unknown): boolean { + try { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const tmp = `${path}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(tmp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); + renameSync(tmp, path); + return true; + } catch { + return false; + } +} + +function readJson(path: string): T | undefined { + if (!existsSync(path)) return undefined; + try { + return JSON.parse(readFileSync(path, "utf8")) as T; + } catch { + return undefined; + } +} + +function stableId(): string { + return `anon_${randomBytes(16).toString("hex")}`; +} + +function ephemeralId(): string { + return `ephemeral_${randomBytes(16).toString("hex")}`; +} + +function isTelemetrySurface(value: unknown): value is TelemetrySurface { + return ( + value === "cli" || + value === "serve" || + value === "attach" || + value === "daemon" || + value === "native" || + value === "code_mode" + ); +} + +function isDeliveryHealth(value: unknown): value is TelemetryDeliveryHealth { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + for (const provider of Object.values(value)) { + if (!provider || typeof provider !== "object" || Array.isArray(provider)) return false; + for (const count of Object.values(provider)) { + if (typeof count !== "number" || !Number.isInteger(count) || count < 0) return false; + } + } + return true; +} diff --git a/packages/core/test/config.test.ts b/packages/core/test/config.test.ts index 20424eeb..4379657e 100644 --- a/packages/core/test/config.test.ts +++ b/packages/core/test/config.test.ts @@ -709,6 +709,70 @@ describe("config", () => { rmSync(dir, { recursive: true, force: true }); }); + it("keeps top-level telemetry as a user-only preference during project config merge", () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-config-")); + const userConfigPath = join(dir, "user", "config.json"); + const projectConfigPath = join(dir, ".caplets", "config.json"); + mkdirSync(join(dir, "user"), { recursive: true }); + mkdirSync(join(dir, ".caplets"), { recursive: true }); + writeFileSync( + userConfigPath, + JSON.stringify({ + telemetry: false, + mcpServers: { + userOnly: { + name: "User Only", + description: "A useful user-only downstream server.", + command: "user-only", + }, + }, + }), + ); + writeFileSync( + projectConfigPath, + JSON.stringify({ + telemetry: true, + mcpServers: { + projectOnly: { + name: "Project Only", + description: "A useful project-only downstream server.", + command: "project-only", + }, + }, + }), + ); + + const config = loadConfig(userConfigPath, projectConfigPath); + + expect(config.telemetry).toBe(false); + expect(Object.keys(config.mcpServers).sort()).toEqual(["projectOnly", "userOnly"]); + rmSync(dir, { recursive: true, force: true }); + }); + + it("ignores project-only telemetry config", () => { + const dir = mkdtempSync(join(tmpdir(), "caplets-config-")); + const projectConfigPath = join(dir, ".caplets", "config.json"); + mkdirSync(join(dir, ".caplets"), { recursive: true }); + writeFileSync( + projectConfigPath, + JSON.stringify({ + telemetry: false, + mcpServers: { + projectOnly: { + name: "Project Only", + description: "A useful project-only downstream server.", + command: "project-only", + }, + }, + }), + ); + + const config = loadConfig(join(dir, "missing-user-config.json"), projectConfigPath); + + expect(config.telemetry).toBeUndefined(); + rmSync(dir, { recursive: true, force: true }); + }); + it("loads top-level and directory Caplet files with project Caplets winning", () => { const dir = mkdtempSync(join(tmpdir(), "caplets-files-")); const userRoot = join(dir, "user"); diff --git a/packages/core/test/native.test.ts b/packages/core/test/native.test.ts index 2aebe875..47ac6663 100644 --- a/packages/core/test/native.test.ts +++ b/packages/core/test/native.test.ts @@ -2,13 +2,14 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createNativeCapletsService, nativeCapletPromptGuidance, nativeCapletToolName, nativeCapletsSystemGuidance, } from "../src/native"; +import { recordTelemetryNoticeShown } from "../src/telemetry"; import { FileVaultStore } from "../src/vault"; const fixturesDir = fileURLToPath(new URL("fixtures", import.meta.url)); @@ -133,6 +134,75 @@ describe("native Caplets service", () => { } }); + it("suppresses native-first telemetry until a visible notice has been recorded", async () => { + const { dir, configPath, projectConfigPath } = tempConfig({ + mcpServers: { + alpha: { + name: "Alpha", + description: "Search alpha project documents.", + command: process.execPath, + }, + }, + }); + dirs.push(dir); + const capture = vi.fn(); + const service = createNativeCapletsService({ + configPath, + projectConfigPath, + telemetryStateDir: join(dir, "state"), + telemetryEnv: {}, + telemetryDispatcher: { capture, shutdown: vi.fn() }, + }); + + try { + await service.execute("alpha", { operation: "inspect" }); + expect(capture).not.toHaveBeenCalled(); + } finally { + await service.close(); + } + }); + + it("captures native tool telemetry after prior visible notice", async () => { + const { dir, configPath, projectConfigPath } = tempConfig({ + mcpServers: { + alpha: { + name: "Alpha", + description: "Search alpha project documents.", + command: process.execPath, + }, + }, + }); + dirs.push(dir); + const stateDir = join(dir, "state"); + recordTelemetryNoticeShown({ stateDir, surface: "cli" }); + const capture = vi.fn(); + const service = createNativeCapletsService({ + configPath, + projectConfigPath, + telemetryStateDir: stateDir, + telemetryEnv: {}, + telemetryDispatcher: { capture, shutdown: vi.fn() }, + }); + + try { + await service.execute("alpha", { operation: "inspect" }); + await expect.poll(() => capture.mock.calls.length).toBe(1); + expect(capture.mock.calls[0]?.[1]).toMatchObject({ + provider: "posthog", + name: "caplets_tool_activation", + properties: expect.objectContaining({ + surface: "native", + command_family: "native", + operation_family: "inspect", + outcome: "success", + integration: "native", + }), + }); + } finally { + await service.close(); + } + }); + it("quarantines Vault-backed Caplets until the configured access grant exists", async () => { const { dir, configPath, projectConfigPath } = tempConfig({ mcpServers: { diff --git a/packages/core/test/telemetry-cli.test.ts b/packages/core/test/telemetry-cli.test.ts new file mode 100644 index 00000000..e23dde43 --- /dev/null +++ b/packages/core/test/telemetry-cli.test.ts @@ -0,0 +1,199 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { runCli } from "../src/cli"; +import { completeCliWords } from "../src/cli/completion"; +import { readTelemetryIdentity, readTelemetryNotice, TelemetryDebugSink } from "../src/telemetry"; + +const dirs: string[] = []; + +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), "caplets-telemetry-cli-")); + dirs.push(dir); + return dir; +} + +describe("telemetry CLI", () => { + afterEach(() => { + for (const dir of dirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reports status with disable instructions and env precedence", async () => { + const dir = tempDir(); + const out: string[] = []; + + await runCli(["telemetry", "status"], { + env: { CAPLETS_DISABLE_TELEMETRY: "1" }, + telemetryStateDir: join(dir, "state"), + writeOut: (value) => out.push(value), + }); + + expect(out.join("")).toContain("Telemetry: disabled"); + expect(out.join("")).toContain("Decision: env"); + expect(out.join("")).toContain("CAPLETS_DISABLE_TELEMETRY=1"); + }); + + it("enable and disable mutate only the user config", async () => { + const dir = tempDir(); + const userConfig = join(dir, "user", "config.json"); + const projectConfig = join(dir, "project", ".caplets", "config.json"); + mkdirSync(join(dir, "user"), { recursive: true }); + mkdirSync(join(dir, "project", ".caplets"), { recursive: true }); + const env = { CAPLETS_CONFIG: userConfig, CAPLETS_PROJECT_CONFIG: projectConfig }; + + await runCli(["telemetry", "disable"], { env, writeOut: () => {} }); + expect(JSON.parse(readFileSync(userConfig, "utf8")).telemetry).toBe(false); + expect(existsSync(projectConfig)).toBe(false); + + await runCli(["telemetry", "enable"], { env, writeOut: () => {} }); + expect(JSON.parse(readFileSync(userConfig, "utf8")).telemetry).toBe(true); + }); + + it("delete-id and rotate-id manage only local identity state", async () => { + const dir = tempDir(); + const stateDir = join(dir, "state"); + const first = readTelemetryIdentity({ stateDir, create: true }); + const out: string[] = []; + + await runCli(["telemetry", "rotate-id"], { + telemetryStateDir: stateDir, + writeOut: (value) => out.push(value), + }); + const rotated = readTelemetryIdentity({ stateDir, create: false }); + expect(rotated.id).not.toBe(first.id); + expect(out.join("")).toContain("does not delete provider-side historical anonymous events"); + + await runCli(["telemetry", "delete-id"], { + telemetryStateDir: stateDir, + writeOut: (value) => out.push(value), + }); + expect(readTelemetryIdentity({ stateDir, create: false }).kind).toBe("ephemeral"); + }); + + it("prints first-run notice to stderr only for eligible TTY commands", async () => { + const dir = tempDir(); + const out: string[] = []; + const err: string[] = []; + + await runCli(["serve"], { + env: {}, + telemetryStateDir: join(dir, "state"), + stderrIsTTY: true, + writeOut: (value) => out.push(value), + writeErr: (value) => err.push(value), + serve: async () => {}, + }); + + expect(out.join("")).toBe(""); + expect(err.join("")).toContain("Caplets collects anonymous telemetry"); + expect(err.join("")).toContain("CAPLETS_DISABLE_TELEMETRY=1"); + expect(readTelemetryNotice({ stateDir: join(dir, "state") }).shown).toBe(true); + + await runCli(["serve"], { + env: {}, + telemetryStateDir: join(dir, "state"), + stderrIsTTY: true, + writeErr: (value) => err.push(value), + serve: async () => {}, + }); + expect(err.join("").match(/Caplets collects anonymous telemetry/gu)).toHaveLength(1); + }); + + it("does not mark notice shown when stderr is redirected", async () => { + const dir = tempDir(); + const err: string[] = []; + + await runCli(["serve"], { + env: {}, + telemetryStateDir: join(dir, "state"), + stderrIsTTY: false, + writeErr: (value) => err.push(value), + serve: async () => {}, + }); + + expect(err.join("")).toBe(""); + expect(readTelemetryNotice({ stateDir: join(dir, "state") }).shown).toBe(false); + }); + + it("does not let enable override CAPLETS_DISABLE_TELEMETRY for current status", async () => { + const dir = tempDir(); + const configPath = join(dir, "config.json"); + const out: string[] = []; + const env = { CAPLETS_CONFIG: configPath, CAPLETS_DISABLE_TELEMETRY: "1" }; + + await runCli(["telemetry", "enable"], { env, writeOut: () => {} }); + await runCli(["telemetry", "status"], { env, writeOut: (value) => out.push(value) }); + + expect(out.join("")).toContain("Telemetry: disabled"); + expect(out.join("")).toContain("Decision: env"); + }); + + it("debug prints local sanitized telemetry events", async () => { + const dir = tempDir(); + const out: string[] = []; + + await runCli(["telemetry", "debug", "--", "setup"], { + telemetryStateDir: join(dir, "state"), + writeOut: (value) => out.push(value), + }); + + expect(out.join("")).toContain('"name": "caplets_cli_command"'); + expect(out.join("")).toContain('"command_family": "setup"'); + expect(out.join("")).not.toContain(dir); + }); + + it("prints the first-run notice for tracked commands without command-local notice calls", async () => { + const dir = tempDir(); + const err: string[] = []; + const configPath = join(dir, "config.json"); + + await runCli(["init", "--global"], { + env: { CAPLETS_CONFIG: configPath }, + telemetryStateDir: join(dir, "state"), + stderrIsTTY: true, + writeOut: () => {}, + writeErr: (value) => err.push(value), + }); + + expect(err.join("")).toContain("Caplets collects anonymous telemetry"); + expect(readTelemetryNotice({ stateDir: join(dir, "state") }).shown).toBe(true); + }); + + it("captures sanitized reliability but no product event for parse errors in debug mode", async () => { + const dir = tempDir(); + const sink = new TelemetryDebugSink(); + + await expect( + runCli(["init", "--typo"], { + env: { CAPLETS_TELEMETRY_DEBUG: "1" }, + telemetryStateDir: join(dir, "state"), + telemetryDebugSink: sink, + writeErr: () => {}, + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + + expect(sink.records).toHaveLength(1); + expect(sink.records[0]?.event).toMatchObject({ + provider: "sentry", + tags: expect.objectContaining({ + command_family: "init", + error_code: "REQUEST_INVALID", + diagnostic_category: "validation", + }), + }); + }); + + it("completion includes telemetry subcommands", async () => { + await expect(completeCliWords(["telemetry", ""])).resolves.toEqual([ + "status", + "enable", + "disable", + "delete-id", + "rotate-id", + "debug", + ]); + }); +}); diff --git a/packages/core/test/telemetry-docs.test.ts b/packages/core/test/telemetry-docs.test.ts new file mode 100644 index 00000000..3b5c56f9 --- /dev/null +++ b/packages/core/test/telemetry-docs.test.ts @@ -0,0 +1,53 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const repoRoot = join(import.meta.dirname, "..", "..", ".."); + +function read(path: string): string { + return readFileSync(join(repoRoot, path), "utf8"); +} + +describe("telemetry product docs", () => { + it("names every provider launch-gate field", () => { + const text = read("docs/product/telemetry-provider-readiness.md"); + + for (const expected of [ + "PostHog project", + "PostHog intake identifier", + "Sentry project", + "Sentry intake identifier", + "Owner", + "Review date", + "Retention", + "Ingestion monitoring", + "Revocation", + "Release Gate", + ]) { + expect(text).toContain(expected); + } + }); + + it("maps decision questions to allowlisted event families", () => { + const text = read("docs/product/telemetry-readout.md"); + + for (const expected of [ + "Where does setup fail?", + "Which surfaces are active?", + "Is local, remote, or cloud runtime worth more investment?", + "Are native integrations used?", + "Which exposure modes are used?", + "Which backend families deserve investment?", + "Is Code Mode succeeding?", + "What reliability pressure is highest?", + "caplets_cli_command", + "caplets_runtime_lifecycle", + "caplets_tool_activation", + "caplets_code_mode_outcome", + "caplets_reliability_error", + "caplets_delivery_health", + ]) { + expect(text).toContain(expected); + } + }); +}); diff --git a/packages/core/test/telemetry-events.test.ts b/packages/core/test/telemetry-events.test.ts new file mode 100644 index 00000000..55325d23 --- /dev/null +++ b/packages/core/test/telemetry-events.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from "vitest"; +import { + buildProductTelemetryEvent, + buildReliabilityTelemetryEvent, + durationBucket, + timeoutBucket, +} from "../src/telemetry"; + +describe("telemetry event builders", () => { + it("builds allowlisted product events with categorical properties", () => { + expect( + buildProductTelemetryEvent({ + name: "caplets_cli_command", + distinctId: "anon_1234567890abcdef1234567890abcdef", + properties: { + package: "@caplets/core", + version: "0.27.0", + surface: "cli", + runtime_mode: "local", + execution_context: "interactive", + command_family: "setup", + outcome: "success", + duration_bucket: "lt_1s", + }, + }), + ).toEqual({ + provider: "posthog", + name: "caplets_cli_command", + distinctId: "anon_1234567890abcdef1234567890abcdef", + properties: { + $process_person_profile: false, + package: "@caplets/core", + version: "0.27.0", + surface: "cli", + runtime_mode: "local", + execution_context: "interactive", + command_family: "setup", + outcome: "success", + duration_bucket: "lt_1s", + }, + }); + }); + + it("rejects unknown event and property keys", () => { + expect(() => + buildProductTelemetryEvent({ + name: "unknown" as never, + distinctId: "anon_1234567890abcdef1234567890abcdef", + properties: { surface: "cli" }, + }), + ).toThrow(/unknown telemetry event/u); + + expect(() => + buildProductTelemetryEvent({ + name: "caplets_cli_command", + distinctId: "anon_1234567890abcdef1234567890abcdef", + properties: { raw_path: "/tmp/secret" } as never, + }), + ).toThrow(/unknown telemetry property/u); + }); + + it("rejects path, URL, hostname, env, token, and id-shaped raw values", () => { + for (const value of [ + "/home/ian/project", + "https://example.com/token", + "api.internal.example.com", + "sk-abc123456789", + "ghp_abcdefghijklmnopqrstuvwxyz123456", + "CAPLETS_REMOTE_URL=https://example.com", + ]) { + expect(() => + buildProductTelemetryEvent({ + name: "caplets_cli_command", + distinctId: "anon_1234567890abcdef1234567890abcdef", + properties: { surface: value } as never, + }), + ).toThrow(/unsafe telemetry property/u); + } + }); + + it("builds reliability events without raw message, stack, or details", () => { + expect( + buildReliabilityTelemetryEvent({ + name: "caplets_reliability_error", + properties: { + package: "@caplets/core", + version: "0.27.0", + surface: "cli", + runtime_mode: "local", + command_family: "serve", + error_code: "CONFIG_INVALID", + diagnostic_category: "config", + os_family: "linux", + arch: "x64", + node_major: 22, + }, + }), + ).toEqual({ + provider: "sentry", + name: "caplets_reliability_error", + fingerprint: ["@caplets/core", "cli", "serve", "local", "CONFIG_INVALID", "config"], + tags: { + package: "@caplets/core", + version: "0.27.0", + surface: "cli", + runtime_mode: "local", + command_family: "serve", + error_code: "CONFIG_INVALID", + diagnostic_category: "config", + os_family: "linux", + arch: "x64", + node_major: "22", + }, + }); + }); + + it("buckets duration and timeout values", () => { + expect(durationBucket(50)).toBe("lt_100ms"); + expect(durationBucket(999)).toBe("lt_1s"); + expect(durationBucket(4_000)).toBe("lt_5s"); + expect(durationBucket(30_000)).toBe("gte_30s"); + expect(timeoutBucket(undefined)).toBe("none"); + expect(timeoutBucket(500)).toBe("lt_1s"); + expect(timeoutBucket(90_000)).toBe("gte_60s"); + }); +}); diff --git a/packages/core/test/telemetry-providers.test.ts b/packages/core/test/telemetry-providers.test.ts new file mode 100644 index 00000000..e41b207a --- /dev/null +++ b/packages/core/test/telemetry-providers.test.ts @@ -0,0 +1,190 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + buildProductTelemetryEvent, + buildReliabilityTelemetryEvent, + createTelemetryDispatcher, + readTelemetryDeliveryHealth, + resolveTelemetryState, +} from "../src/telemetry"; + +const roots: string[] = []; + +function tempRoot(): string { + const root = join(mkdtempSync(join(tmpdir(), "caplets-telemetry-providers-")), "state"); + roots.push(root); + return root; +} + +describe("telemetry providers", () => { + afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("does not construct SDK clients for disabled, test, or debug state", async () => { + const factory = vi.fn(); + const dispatcher = createTelemetryDispatcher({ + posthogToken: "ph_project", + sentryDsn: "https://public@sentry.example/1", + factories: { createPostHog: factory, createSentry: factory }, + }); + const product = buildProductTelemetryEvent({ + name: "caplets_cli_command", + distinctId: "anon_1234567890abcdef1234567890abcdef", + properties: { surface: "cli", execution_context: "interactive" }, + }); + + await dispatcher.capture( + resolveTelemetryState({ + stateDir: tempRoot(), + env: { CAPLETS_DISABLE_TELEMETRY: "1" }, + surface: "cli", + visibility: "visible", + }), + product, + ); + await dispatcher.capture( + resolveTelemetryState({ + stateDir: tempRoot(), + env: { VITEST: "true" }, + surface: "cli", + visibility: "visible", + }), + product, + ); + await dispatcher.capture( + resolveTelemetryState({ + stateDir: tempRoot(), + env: {}, + surface: "cli", + visibility: "visible", + debug: true, + }), + product, + ); + + expect(factory).not.toHaveBeenCalled(); + }); + + it("captures PostHog events with anonymous profile and GeoIP disabled", async () => { + const capture = vi.fn(); + const stateDir = tempRoot(); + const dispatcher = createTelemetryDispatcher({ + posthogToken: "ph_project", + sentryDsn: "", + factories: { + createPostHog: vi.fn(() => ({ capture, shutdown: vi.fn() })), + }, + }); + const state = resolveTelemetryState({ + stateDir, + env: { CI: "true" }, + surface: "cli", + visibility: "hidden", + }); + + await dispatcher.capture( + state, + buildProductTelemetryEvent({ + name: "caplets_cli_command", + distinctId: state.identity!.id, + properties: { surface: "cli", execution_context: "ci" }, + }), + ); + + expect(capture).toHaveBeenCalledWith({ + distinctId: state.identity!.id, + event: "caplets_cli_command", + properties: { + $geoip_disable: true, + $process_person_profile: false, + surface: "cli", + execution_context: "ci", + }, + }); + }); + + it("captures Sentry reliability events with categorical tags and fingerprint", async () => { + const captureEvent = vi.fn(); + const dispatcher = createTelemetryDispatcher({ + posthogToken: "", + sentryDsn: "https://public@sentry.example/1", + factories: { + createSentry: vi.fn(() => ({ captureEvent, flush: vi.fn() })), + }, + }); + const state = resolveTelemetryState({ + stateDir: tempRoot(), + env: { CI: "true" }, + surface: "cli", + visibility: "hidden", + }); + + await dispatcher.capture( + state, + buildReliabilityTelemetryEvent({ + name: "caplets_reliability_error", + properties: { + package: "@caplets/core", + surface: "cli", + command_family: "serve", + runtime_mode: "local", + error_code: "CONFIG_INVALID", + diagnostic_category: "config", + }, + }), + ); + + expect(captureEvent).toHaveBeenCalledWith({ + level: "error", + tags: { + package: "@caplets/core", + surface: "cli", + command_family: "serve", + runtime_mode: "local", + error_code: "CONFIG_INVALID", + diagnostic_category: "config", + }, + fingerprint: ["@caplets/core", "cli", "serve", "local", "CONFIG_INVALID", "config"], + }); + }); + + it("records delivery health instead of throwing when provider send fails", async () => { + const stateDir = tempRoot(); + const dispatcher = createTelemetryDispatcher({ + posthogToken: "ph_project", + sentryDsn: "", + factories: { + createPostHog: vi.fn(() => ({ + capture: vi.fn(() => { + throw new Error("network"); + }), + shutdown: vi.fn(), + })), + }, + }); + const state = resolveTelemetryState({ + stateDir, + env: { CI: "true" }, + surface: "cli", + visibility: "hidden", + }); + + await dispatcher.capture( + state, + buildProductTelemetryEvent({ + name: "caplets_cli_command", + distinctId: state.identity!.id, + properties: { surface: "cli", execution_context: "ci" }, + }), + ); + + expect(readTelemetryDeliveryHealth({ stateDir })).toEqual({ + posthog: { send_failed: 1 }, + }); + }); +}); diff --git a/packages/core/test/telemetry-redaction.test.ts b/packages/core/test/telemetry-redaction.test.ts new file mode 100644 index 00000000..a00ef120 --- /dev/null +++ b/packages/core/test/telemetry-redaction.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { assertTelemetrySafeProperties, stripSentryEvent } from "../src/telemetry"; + +describe("telemetry privacy guard", () => { + it("rejects unsafe string values before provider calls", () => { + expect(() => + assertTelemetrySafeProperties({ + surface: "cli", + command_family: "setup", + path_like: "/Users/alex/.config/caplets/config.json", + } as never), + ).toThrow(/unknown telemetry property/u); + + expect(() => + assertTelemetrySafeProperties({ + surface: "cli", + command_family: "setup", + error_code: "TOKEN=secret", + }), + ).toThrow(/unsafe telemetry property/u); + + expect(() => + assertTelemetrySafeProperties({ + surface: "cli", + command_family: "workspace-slug", + } as never), + ).toThrow(/unsafe telemetry property/u); + }); + + it("strips Sentry event message, exception, breadcrumbs, request, and extra data", () => { + expect( + stripSentryEvent({ + message: "Raw /tmp/path message", + exception: { values: [{ stacktrace: { frames: [{ filename: "/tmp/x.ts" }] } }] }, + request: { url: "https://example.com" }, + breadcrumbs: [{ message: "secret" }], + extra: { args: ["--token", "secret"] }, + tags: { surface: "cli" }, + fingerprint: ["@caplets/core", "cli"], + }), + ).toEqual({ + tags: { surface: "cli" }, + fingerprint: ["@caplets/core", "cli"], + }); + }); +}); diff --git a/packages/core/test/telemetry-release.test.ts b/packages/core/test/telemetry-release.test.ts new file mode 100644 index 00000000..16c31566 --- /dev/null +++ b/packages/core/test/telemetry-release.test.ts @@ -0,0 +1,46 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { checkTelemetryReleaseEnv } from "../../../scripts/check-telemetry-release-env"; + +const repoRoot = join(import.meta.dirname, "..", "..", ".."); + +function read(path: string): string { + return readFileSync(join(repoRoot, path), "utf8"); +} + +describe("telemetry release environment", () => { + it("requires PostHog and Sentry intake identifiers", () => { + expect(checkTelemetryReleaseEnv({})).toEqual([ + "CAPLETS_POSTHOG_TOKEN is required for telemetry-enabled releases.", + "CAPLETS_SENTRY_DSN is required for telemetry-enabled releases.", + ]); + + expect( + checkTelemetryReleaseEnv({ + CAPLETS_POSTHOG_TOKEN: "phc_test_project_token", + CAPLETS_SENTRY_DSN: "https://public@example.ingest.sentry.io/123", + }), + ).toEqual([]); + }); + + it("rejects placeholders and malformed DSNs", () => { + expect( + checkTelemetryReleaseEnv({ + CAPLETS_POSTHOG_TOKEN: "TODO", + CAPLETS_SENTRY_DSN: "not-a-dsn", + }), + ).toEqual([ + "CAPLETS_POSTHOG_TOKEN must be a valid PostHog project token; placeholders are not allowed.", + "CAPLETS_SENTRY_DSN must be a valid Sentry DSN; placeholders are not allowed.", + ]); + }); + + it("wires telemetry secrets into the release workflow", () => { + const workflow = read(".github/workflows/release.yml"); + + expect(workflow).toContain("pnpm telemetry:check-release-env"); + expect(workflow).toContain("CAPLETS_POSTHOG_TOKEN: ${{ secrets.CAPLETS_POSTHOG_TOKEN }}"); + expect(workflow).toContain("CAPLETS_SENTRY_DSN: ${{ secrets.CAPLETS_SENTRY_DSN }}"); + }); +}); diff --git a/packages/core/test/telemetry-state.test.ts b/packages/core/test/telemetry-state.test.ts new file mode 100644 index 00000000..f15459bb --- /dev/null +++ b/packages/core/test/telemetry-state.test.ts @@ -0,0 +1,181 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { parseConfig } from "../src/config"; +import { + defaultTelemetryDeliveryHealthPath, + defaultTelemetryIdentityPath, + defaultTelemetryNoticePath, +} from "../src/config/paths"; +import { + readTelemetryDeliveryHealth, + readTelemetryIdentity, + readTelemetryNotice, + recordTelemetryDrop, + recordTelemetryNoticeShown, + resolveTelemetryState, + rotateTelemetryIdentity, +} from "../src/telemetry"; + +const roots: string[] = []; + +function tempRoot(): string { + const root = join(mkdtempSync(join(tmpdir(), "caplets-telemetry-")), "state"); + mkdirSync(root, { recursive: true }); + roots.push(root); + return root; +} + +describe("telemetry state", () => { + afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("resolves state paths under Caplets state", () => { + const home = "/home/alex"; + expect(defaultTelemetryIdentityPath({}, home, "linux")).toBe( + join(home, ".local", "state", "caplets", "telemetry", "identity.json"), + ); + expect(defaultTelemetryNoticePath({}, home, "linux")).toBe( + join(home, ".local", "state", "caplets", "telemetry", "notice.json"), + ); + expect(defaultTelemetryDeliveryHealthPath({}, home, "linux")).toBe( + join(home, ".local", "state", "caplets", "telemetry", "delivery-health.json"), + ); + }); + + it("uses a stable random identity and preserves it across disablement", () => { + const stateDir = tempRoot(); + const first = readTelemetryIdentity({ stateDir, create: true }); + const second = readTelemetryIdentity({ stateDir, create: true }); + + expect(first.kind).toBe("stable"); + expect(first.id).toMatch(/^anon_[a-f0-9]{32}$/u); + expect(second.id).toBe(first.id); + + const disabled = resolveTelemetryState({ + config: parseConfig({ telemetry: false }), + stateDir, + env: {}, + surface: "cli", + visibility: "visible", + }); + + expect(disabled.status).toBe("disabled"); + expect(readTelemetryIdentity({ stateDir, create: false }).id).toBe(first.id); + }); + + it("rotates identity with owner-only file permissions where supported", () => { + const stateDir = tempRoot(); + const first = readTelemetryIdentity({ stateDir, create: true }); + const second = rotateTelemetryIdentity({ stateDir }); + + expect(second.id).not.toBe(first.id); + if (process.platform !== "win32") { + expect(statSync(join(stateDir, "identity.json")).mode & 0o777).toBe(0o600); + } + }); + + it("records visible notice only when explicitly marked", () => { + const stateDir = tempRoot(); + + expect(readTelemetryNotice({ stateDir }).shown).toBe(false); + recordTelemetryNoticeShown({ stateDir, surface: "cli" }); + + const notice = readTelemetryNotice({ stateDir }); + expect(notice.shown).toBe(true); + if (!notice.shown) throw new Error("expected notice to be shown"); + expect(notice.surface).toBe("cli"); + }); + + it("disables telemetry when env, config, or test detection says so", () => { + const base = { + stateDir: tempRoot(), + surface: "cli" as const, + visibility: "visible" as const, + }; + + expect( + resolveTelemetryState({ ...base, env: { CAPLETS_DISABLE_TELEMETRY: "1" } }).decider, + ).toBe("env"); + expect( + resolveTelemetryState({ ...base, config: parseConfig({ telemetry: false }), env: {} }) + .decider, + ).toBe("config"); + expect(resolveTelemetryState({ ...base, env: { VITEST: "true" } }).decider).toBe("test"); + }); + + it("suppresses hidden native and daemon surfaces until notice exists", () => { + const stateDir = tempRoot(); + const beforeNotice = resolveTelemetryState({ + stateDir, + env: {}, + surface: "native", + visibility: "hidden", + }); + expect(beforeNotice.status).toBe("suppressed"); + expect(beforeNotice.decider).toBe("notice"); + + recordTelemetryNoticeShown({ stateDir, surface: "cli" }); + + const afterNotice = resolveTelemetryState({ + stateDir, + env: {}, + surface: "native", + visibility: "hidden", + }); + expect(afterNotice.status).toBe("enabled"); + }); + + it("classifies CI without mutating visible notice and falls back to ephemeral identity", () => { + const stateDir = tempRoot(); + const state = resolveTelemetryState({ + stateDir, + env: { CI: "true" }, + surface: "cli", + visibility: "hidden", + }); + + expect(state.status).toBe("enabled"); + expect(state.executionContext).toBe("ci"); + expect(readTelemetryNotice({ stateDir }).shown).toBe(false); + + const readonlyState = join("/dev/null", "caplets-telemetry-state"); + const identity = readTelemetryIdentity({ stateDir: readonlyState, create: true }); + expect(identity.kind).toBe("ephemeral"); + }); + + it("tracks local delivery health without throwing", () => { + const stateDir = tempRoot(); + + recordTelemetryDrop({ stateDir, provider: "posthog", reason: "send_failed" }); + recordTelemetryDrop({ stateDir, provider: "posthog", reason: "send_failed" }); + recordTelemetryDrop({ stateDir, provider: "sentry", reason: "disabled" }); + + expect(readTelemetryDeliveryHealth({ stateDir })).toEqual({ + posthog: { send_failed: 2 }, + sentry: { disabled: 1 }, + }); + expect(JSON.parse(readFileSync(join(stateDir, "delivery-health.json"), "utf8"))).toEqual({ + posthog: { send_failed: 2 }, + sentry: { disabled: 1 }, + }); + }); + + it("does not create state files while merely resolving disabled telemetry", () => { + const stateDir = tempRoot(); + + resolveTelemetryState({ + config: parseConfig({ telemetry: false }), + stateDir, + env: {}, + surface: "cli", + visibility: "visible", + }); + + expect(existsSync(join(stateDir, "identity.json"))).toBe(false); + }); +}); diff --git a/packages/opencode/README.md b/packages/opencode/README.md index d7fd9be4..b5de498b 100644 --- a/packages/opencode/README.md +++ b/packages/opencode/README.md @@ -61,3 +61,11 @@ export default { ``` Plugin config overrides environment variables. The explicit config shape is `{ mode, remote: { url, pollIntervalMs } }`; credentials come from `caplets remote login `. + +## Anonymous Telemetry + +Caplets native integrations share the core anonymous telemetry controls. Native-first runs do not +send telemetry until the CLI has recorded a visible telemetry notice. Disable telemetry with +top-level `"telemetry": false` in the user Caplets config or `CAPLETS_DISABLE_TELEMETRY=1`. +Telemetry never includes prompts, tool arguments, tool outputs, paths, URLs, hostnames, Caplet IDs, +credentials, tokens, raw env, Code Mode code, logs, raw error messages, or unsanitized stack traces. diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 013fc3ab..5b2a112a 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -9,9 +9,10 @@ import { createCapletsOpenCodeHooks } from "./hooks"; export type CapletsOpenCodeConfig = Pick; const plugin = (async (_ctx: PluginInput, config?: CapletsOpenCodeConfig) => { - const service = createNativeCapletsService( - normalizeOpenCodeConfig(config) as NativeCapletsServiceOptions, - ); + const service = createNativeCapletsService({ + ...normalizeOpenCodeConfig(config), + telemetryIntegration: "opencode", + } as NativeCapletsServiceOptions); registerNativeCapletsProcessCleanup(service); if (!(await service.reload())) { throw new Error("Failed to initialize Caplets native service."); diff --git a/packages/opencode/test/opencode.test.ts b/packages/opencode/test/opencode.test.ts index 6ccedd43..b2a62686 100644 --- a/packages/opencode/test/opencode.test.ts +++ b/packages/opencode/test/opencode.test.ts @@ -316,6 +316,7 @@ describe("@caplets/opencode", () => { url: "https://caplets.example.com", pollIntervalMs: 5_000, }, + telemetryIntegration: "opencode", }); }); @@ -342,6 +343,7 @@ describe("@caplets/opencode", () => { expect(nativeMocks.createNativeCapletsService).toHaveBeenCalledWith({ mode: "cloud", remote: { url: "https://cloud.caplets.dev" }, + telemetryIntegration: "opencode", }); }); diff --git a/packages/pi/README.md b/packages/pi/README.md index b34e1659..9085bf5b 100644 --- a/packages/pi/README.md +++ b/packages/pi/README.md @@ -91,3 +91,11 @@ export default createCapletsPiExtension({ The explicit config shape is `{ mode, remote: { url, pollIntervalMs } }`. Credentials come from `caplets remote login `, not settings files or source code. + +## Anonymous Telemetry + +Caplets native integrations share the core anonymous telemetry controls. Native-first runs do not +send telemetry until the CLI has recorded a visible telemetry notice. Disable telemetry with +top-level `"telemetry": false` in the user Caplets config or `CAPLETS_DISABLE_TELEMETRY=1`. +Telemetry never includes prompts, tool arguments, tool outputs, paths, URLs, hostnames, Caplet IDs, +credentials, tokens, raw env, Code Mode code, logs, raw error messages, or unsanitized stack traces. diff --git a/packages/pi/src/index.ts b/packages/pi/src/index.ts index ccb21c1d..1c624779 100644 --- a/packages/pi/src/index.ts +++ b/packages/pi/src/index.ts @@ -111,7 +111,11 @@ async function registerCapletsPiExtension( : undefined; const serviceOptions = explicitNativeOptions ?? settingsArgs ?? {}; const service = - options.service ?? createNativeCapletsService(nativeServiceOptions(serviceOptions)); + options.service ?? + createNativeCapletsService({ + ...nativeServiceOptions(serviceOptions), + telemetryIntegration: "pi", + }); const showStatusWidget = shouldShowStatusWidget( serviceOptions, options.statusWidget ?? settingsArgs?.statusWidget, diff --git a/packages/pi/test/pi.test.ts b/packages/pi/test/pi.test.ts index 7f7f13ed..e96e536e 100644 --- a/packages/pi/test/pi.test.ts +++ b/packages/pi/test/pi.test.ts @@ -720,7 +720,9 @@ describe("@caplets/pi", () => { await capletsPiExtension({ registerTool: vi.fn() }); - expect(nativeMocks.createNativeCapletsService).toHaveBeenCalledWith({}); + expect(nativeMocks.createNativeCapletsService).toHaveBeenCalledWith({ + telemetryIntegration: "pi", + }); expect(nativeMocks.registerNativeCapletsProcessCleanup).toHaveBeenCalledWith(service); }); @@ -734,7 +736,10 @@ describe("@caplets/pi", () => { createCapletsPiExtension({ args })({ registerTool: vi.fn() }); - expect(nativeMocks.createNativeCapletsService).toHaveBeenCalledWith(args); + expect(nativeMocks.createNativeCapletsService).toHaveBeenCalledWith({ + ...args, + telemetryIntegration: "pi", + }); expect(nativeMocks.registerNativeCapletsProcessCleanup).toHaveBeenCalledWith(service); }); @@ -1048,6 +1053,7 @@ describe("@caplets/pi", () => { expect(nativeMocks.createNativeCapletsService).toHaveBeenLastCalledWith({ mode: "remote", remote: { url: "http://localhost:5387" }, + telemetryIntegration: "pi", }); }); @@ -1160,6 +1166,7 @@ describe("@caplets/pi", () => { url: "https://caplets.example.com", pollIntervalMs: 1_000, }, + telemetryIntegration: "pi", }); }); @@ -1203,7 +1210,9 @@ describe("@caplets/pi", () => { await capletsPiExtension(api as unknown as PiExtensionApi); - expect(nativeMocks.createNativeCapletsService).toHaveBeenLastCalledWith({}); + expect(nativeMocks.createNativeCapletsService).toHaveBeenLastCalledWith({ + telemetryIntegration: "pi", + }); }); it("default export falls back to empty args when Pi settings are missing", async () => { @@ -1216,7 +1225,9 @@ describe("@caplets/pi", () => { await capletsPiExtension(api as unknown as PiExtensionApi); - expect(nativeMocks.createNativeCapletsService).toHaveBeenLastCalledWith({}); + expect(nativeMocks.createNativeCapletsService).toHaveBeenLastCalledWith({ + telemetryIntegration: "pi", + }); }); it("warns and falls back to empty args when Pi settings are malformed", async () => { @@ -1350,7 +1361,10 @@ describe("@caplets/pi", () => { await createCapletsPiExtension({ args: { mode: "local" } })(api as unknown as PiExtensionApi); expect(fsMocks.readFile).not.toHaveBeenCalled(); - expect(nativeMocks.createNativeCapletsService).toHaveBeenLastCalledWith({ mode: "local" }); + expect(nativeMocks.createNativeCapletsService).toHaveBeenLastCalledWith({ + mode: "local", + telemetryIntegration: "pi", + }); }); it("closes owned services on Pi session shutdown", async () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a5767ab6..23bdadf2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,7 +22,7 @@ importers: version: 7.0.0-dev.20260613.1 alchemy: specifier: 0.93.12 - version: 0.93.12(@opentelemetry/api@1.9.0)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0))(workerd@1.20260617.1) + version: 0.93.12(@opentelemetry/api@1.9.1)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0))(workerd@1.20260617.1) husky: specifier: ^9.1.7 version: 9.1.7 @@ -52,7 +52,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.1.8 - version: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) apps/docs: dependencies: @@ -151,7 +151,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.1.8 - version: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) packages/cli: dependencies: @@ -176,7 +176,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.1.8 - version: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) packages/core: dependencies: @@ -198,6 +198,9 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.29.0 version: 1.29.0(zod@4.4.3) + '@sentry/node': + specifier: ^10.60.0 + version: 10.60.0(@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1)) '@ungap/structured-clone': specifier: ^1.3.1 version: 1.3.1 @@ -219,6 +222,9 @@ importers: hono: specifier: ^4.12.25 version: 4.12.26 + posthog-node: + specifier: ^4.18.0 + version: 4.18.0 quickjs-emscripten: specifier: ^0.32.0 version: 0.32.0 @@ -255,7 +261,7 @@ importers: version: 1.1.2 vitest: specifier: ^4.1.8 - version: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) packages/opencode: dependencies: @@ -283,7 +289,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.1.8 - version: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) packages/pi: dependencies: @@ -343,6 +349,17 @@ packages: peerDependencies: openapi-types: '>=7' + '@apm-js-collab/code-transformer-bundler-plugins@0.5.0': + resolution: {integrity: sha512-YxLBY5nGlurL7QeJLq6e5g0ouBpAp0pwgyA/5rHXEXwhiPLn9ZHbT+Y2LlP90GT872cSocfjWRYu/fnpuBudNQ==} + engines: {node: '>=18.0.0'} + + '@apm-js-collab/code-transformer@0.15.0': + resolution: {integrity: sha512-XmXYVs8CzJ1Aj79noVbn2weUO/XWtRyURpGqx7aU7DOXlUQhR0WKOQNF0okh7PCeY37vxf7kU3v57OAkEPm3ww==} + hasBin: true + + '@apm-js-collab/tracing-hooks@0.10.0': + resolution: {integrity: sha512-2/Z3NTewJTruUkmsSnBC5bJlLNUd9keuD1OLlTEpim4FyLhm6m2Rnfv+wrFdUvFfhmH8CRdiDZBqBrn+wyaGuA==} + '@astrojs/check@0.9.9': resolution: {integrity: sha512-A5UW8uIuErLWEoRQvzgXpO1gTjUFtK8r7nU2Z7GewAMxUb7bPvpk11qaKKgxqXlHJWlAvaaxy+Xg28A6bmQ1Tg==} hasBin: true @@ -1693,10 +1710,42 @@ packages: '@opencode-ai/sdk@1.17.9': resolution: {integrity: sha512-MHmXEpGPHkg14v1p+cUlIOUxd6DQdSElfau9nqY7tcDI0x5r4Y8D0dKXcyAh0Gc73ptaGW67Vg84nkcV6O27Pw==} + '@opentelemetry/api-logs@0.214.0': + resolution: {integrity: sha512-40lSJeqYO8Uz2Yj7u94/SJWE/wONa7rmMKjI1ZcIjgf3MHNHv1OZUCrCETGuaRF62d5pQD1wKIW+L4lmSMTzZA==} + engines: {node: '>=8.0.0'} + '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/core@2.8.0': + resolution: {integrity: sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/instrumentation@0.214.0': + resolution: {integrity: sha512-MHqEX5Dk59cqVah5LiARMACku7jXSVk9iVDWOea4x3cr7VfdByeDCURK6o1lntT1JS/Tsovw01UJrBhN3/uC5w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/resources@2.8.0': + resolution: {integrity: sha512-qmXQ27ilDbUK/vGMqwL8D4/rhn76C+sherM4wTbjlfknR8Nvfc/hCxjRJPhkzZzUsPiNg16SA31NxMabwttRjg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@2.8.0': + resolution: {integrity: sha512-mhU4jp+vW0mGbFRd+GeXHvmfA4aDqWjBjLC3pE5XMpLs0IE2ryYb019Ts2AQrOq67gaTF25D91+fgvEHDZEnuQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/semantic-conventions@1.41.1': resolution: {integrity: sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA==} engines: {node: '>=14'} @@ -2277,6 +2326,51 @@ packages: '@sec-ant/readable-stream@0.4.1': resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} + '@sentry/conventions@0.12.0': + resolution: {integrity: sha512-z1JQrl/1SLY+8wpzvork6vl+fpsg/oCCxM7HWWhUnI/R+OGNyoIzieQuggX3uUMY7NBtp8UWCQx6FeFazzOF9g==} + engines: {node: '>=14'} + + '@sentry/core@10.60.0': + resolution: {integrity: sha512-szN7ccOJAEaLb1BBQzCQhABGMTJmKNUk0G2sc7rWhajeXoZoMKIbNkI9RvJrFuV69cbad/d/BKGBjbpJhySAzw==} + engines: {node: '>=18'} + + '@sentry/node-core@10.60.0': + resolution: {integrity: sha512-aXi9ixvP+hgUZPPZCRwMNHgY2I0gkSeoAKAUuysDJhWDmrygwfGdlkbGmmtW6PQjtMYFx69Igt5btvhjEBoJTw==} + engines: {node: '>=18'} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + '@opentelemetry/core': ^1.30.1 || ^2.1.0 + '@opentelemetry/exporter-trace-otlp-http': '>=0.57.0 <1' + '@opentelemetry/instrumentation': '>=0.57.1 <1' + '@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + '@opentelemetry/core': + optional: true + '@opentelemetry/exporter-trace-otlp-http': + optional: true + '@opentelemetry/instrumentation': + optional: true + '@opentelemetry/sdk-trace-base': + optional: true + + '@sentry/node@10.60.0': + resolution: {integrity: sha512-u//paUrkKaCr0oNn7r7UulGydkYMSkU1wQOIpG/P/jf7psZWnyXhgeszHzUfZXo6pCdxXG9z9viPvzGjqPQN7A==} + engines: {node: '>=18'} + + '@sentry/opentelemetry@10.60.0': + resolution: {integrity: sha512-gl+2NVH+9RmTu7pd9kV1tKif+Th+p9tmnXR1l3Sb3Wqo1ir5FaNMKrloWEKMXjnepii9EJUrEHdSC+i8NoexxQ==} + engines: {node: '>=18'} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + '@opentelemetry/core': ^1.30.1 || ^2.1.0 + '@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0 + + '@sentry/server-utils@10.60.0': + resolution: {integrity: sha512-SX+MzWM3nz5ttKT48rlfktm0ERyIpDLma+b6pYeWgW2oFHKcpIu0g0qMGJrZs4lKM3MlgV7IqLa4texMqTp9kQ==} + engines: {node: '>=18'} + '@shikijs/core@4.2.0': resolution: {integrity: sha512-Hc87Ab1Ld/vEbZRCbwx344I5v+4RU8CVToUTRkqXL1+TjbuOp9U5Xa0M23V4GEWHxVn+yO5otb+HkQVm3ptWQQ==} engines: {node: '>=20'} @@ -2685,6 +2779,11 @@ packages: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} + acorn-import-attributes@1.9.5: + resolution: {integrity: sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==} + peerDependencies: + acorn: ^8 + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -2695,6 +2794,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + agent-base@7.1.4: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} @@ -2844,9 +2947,15 @@ packages: engines: {node: '>=22.12.0', npm: '>=9.6.5', pnpm: '>=7.1.0'} hasBin: true + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + aws4fetch@1.0.20: resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==} + axios@1.18.1: + resolution: {integrity: sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==} + axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} @@ -2968,6 +3077,9 @@ packages: resolution: {integrity: sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==} engines: {node: '>=8'} + cjs-module-lexer@2.2.0: + resolution: {integrity: sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==} + cli-cursor@5.0.0: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} @@ -2994,6 +3106,10 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} @@ -3115,6 +3231,10 @@ packages: defu@6.1.7: resolution: {integrity: sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==} + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -3335,6 +3455,10 @@ packages: resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} engines: {node: '>= 0.4'} + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + esast-util-from-estree@2.0.0: resolution: {integrity: sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==} @@ -3372,6 +3496,14 @@ packages: engines: {node: '>=4'} hasBin: true + esquery@1.7.0: + resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} + engines: {node: '>=0.10'} + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + estree-util-attach-comments@3.0.0: resolution: {integrity: sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw==} @@ -3527,6 +3659,15 @@ packages: resolution: {integrity: sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ==} engines: {node: '>=8'} + follow-redirects@1.16.0: + resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + fontace@0.4.1: resolution: {integrity: sha512-lDMvbAzSnHmbYMTEld5qdtvNH2/pWpICOqpean9IgC7vUbUJc3k+k5Dokp85CegamqQpFbXf0rAVkbzpyTA8aw==} @@ -3538,6 +3679,10 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} + engines: {node: '>= 6'} + formdata-node@6.0.3: resolution: {integrity: sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==} engines: {node: '>= 18'} @@ -3652,6 +3797,10 @@ packages: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + hasown@2.0.4: resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} engines: {node: '>= 0.4'} @@ -3759,6 +3908,10 @@ packages: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + https-proxy-agent@7.0.6: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} @@ -3799,6 +3952,10 @@ packages: immediate@3.0.6: resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==} + import-in-the-middle@3.2.0: + resolution: {integrity: sha512-vR2B6HKIhaBjcZr2bLpFiJ1VbzOlRQ7aby4/gw5WPIzToLjqpfWw3VJ4sk1uDchoOODEirvO2jyrSPtUSL5CrQ==} + engines: {node: '>=18'} + inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -4180,6 +4337,10 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} + meriyah@6.1.4: + resolution: {integrity: sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ==} + engines: {node: '>=18.0.0'} + meshoptimizer@1.1.1: resolution: {integrity: sha512-oRFNWJRDA/WTrVj7NWvqa5HqE1t9MYDj2VaWirQCzCCrAd2GHrqR/sQezCxiWATPNlKTcRaPRHPJwIRoPBAp5g==} @@ -4295,10 +4456,18 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + mime-db@1.54.0: resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} engines: {node: '>= 0.6'} + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + mime-types@3.0.2: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} @@ -4333,6 +4502,9 @@ packages: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} + module-details-from-path@1.0.4: + resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} + mri@1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} @@ -4638,6 +4810,10 @@ packages: resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} engines: {node: ^10 || ^12 || >=14} + posthog-node@4.18.0: + resolution: {integrity: sha512-XROs1h+DNatgKh/AlIlCtDxWzwrKdYDb2mOs58n4yN8BkGN9ewqeQwG5ApS4/IzwCb7HPttUkOVulkYatd2PIw==} + engines: {node: '>=15.0.0'} + prettier-plugin-astro@0.14.1: resolution: {integrity: sha512-RiBETaaP9veVstE4vUwSIcdATj6dKmXljouXc/DDNwBSPTp8FRkLGDSGFClKsAFeeg+13SB0Z1JZvbD76bigJw==} engines: {node: ^14.15.0 || >=16.0.0} @@ -4677,6 +4853,10 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} + proxy-from-env@2.1.0: + resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} + engines: {node: '>=10'} + punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -4807,6 +4987,10 @@ packages: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} + require-in-the-middle@8.0.1: + resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==} + engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'} + resolve-from@5.0.0: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} @@ -4885,6 +5069,9 @@ packages: resolution: {integrity: sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==} engines: {node: '>=11.0.0'} + semifies@1.0.0: + resolution: {integrity: sha512-xXR3KGeoxTNWPD4aBvL5NUpMTT7WMANr3EWnaS190QVkY52lqqcVRD7Q05UVbBhiWDGWMlJEUam9m7uFFGVScw==} + semver@7.8.0: resolution: {integrity: sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==} engines: {node: '>=10'} @@ -4982,6 +5169,10 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + source-map@0.7.6: resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} engines: {node: '>= 12'} @@ -5738,6 +5929,30 @@ snapshots: call-me-maybe: 1.0.2 openapi-types: 12.1.3 + '@apm-js-collab/code-transformer-bundler-plugins@0.5.0': + dependencies: + '@apm-js-collab/code-transformer': 0.15.0 + es-module-lexer: 2.1.0 + magic-string: 0.30.21 + module-details-from-path: 1.0.4 + + '@apm-js-collab/code-transformer@0.15.0': + dependencies: + '@types/estree': 1.0.9 + astring: 1.9.0 + esquery: 1.7.0 + meriyah: 6.1.4 + semifies: 1.0.0 + source-map: 0.6.1 + + '@apm-js-collab/tracing-hooks@0.10.0': + dependencies: + '@apm-js-collab/code-transformer': 0.15.0 + debug: 4.4.3 + module-details-from-path: 1.0.4 + transitivePeerDependencies: + - supports-color + '@astrojs/check@0.9.9(prettier-plugin-astro@0.14.1)(prettier@3.8.4)(typescript@6.0.3)': dependencies: '@astrojs/language-server': 2.16.10(prettier-plugin-astro@0.14.1)(prettier@3.8.4)(typescript@6.0.3) @@ -7177,8 +7392,41 @@ snapshots: dependencies: cross-spawn: 7.0.6 + '@opentelemetry/api-logs@0.214.0': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api@1.9.0': {} + '@opentelemetry/api@1.9.1': {} + + '@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/semantic-conventions': 1.41.1 + + '@opentelemetry/instrumentation@0.214.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.214.0 + import-in-the-middle: 3.2.0 + require-in-the-middle: 8.0.1 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/resources@2.8.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.41.1 + + '@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.41.1 + '@opentelemetry/semantic-conventions@1.41.1': {} '@oslojs/encoding@1.1.0': {} @@ -7493,6 +7741,57 @@ snapshots: '@sec-ant/readable-stream@0.4.1': {} + '@sentry/conventions@0.12.0': {} + + '@sentry/core@10.60.0': {} + + '@sentry/node-core@10.60.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1))(@opentelemetry/instrumentation@0.214.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1))': + dependencies: + '@sentry/conventions': 0.12.0 + '@sentry/core': 10.60.0 + '@sentry/opentelemetry': 10.60.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1)) + import-in-the-middle: 3.2.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.214.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.8.0(@opentelemetry/api@1.9.1) + + '@sentry/node@10.60.0(@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1))': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.214.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.41.1 + '@sentry/core': 10.60.0 + '@sentry/node-core': 10.60.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1))(@opentelemetry/instrumentation@0.214.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1)) + '@sentry/opentelemetry': 10.60.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1)) + '@sentry/server-utils': 10.60.0 + import-in-the-middle: 3.2.0 + transitivePeerDependencies: + - '@opentelemetry/core' + - '@opentelemetry/exporter-trace-otlp-http' + - supports-color + + '@sentry/opentelemetry@10.60.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1))': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.8.0(@opentelemetry/api@1.9.1) + '@sentry/conventions': 0.12.0 + '@sentry/core': 10.60.0 + + '@sentry/server-utils@10.60.0': + dependencies: + '@apm-js-collab/code-transformer': 0.15.0 + '@apm-js-collab/code-transformer-bundler-plugins': 0.5.0 + '@apm-js-collab/tracing-hooks': 0.10.0 + '@sentry/conventions': 0.12.0 + '@sentry/core': 10.60.0 + magic-string: 0.30.21 + transitivePeerDependencies: + - supports-color + '@shikijs/core@4.2.0': dependencies: '@shikijs/primitive': 4.2.0 @@ -7905,12 +8204,22 @@ snapshots: mime-types: 3.0.2 negotiator: 1.0.0 + acorn-import-attributes@1.9.5(acorn@8.17.0): + dependencies: + acorn: 8.17.0 + acorn-jsx@5.3.2(acorn@8.17.0): dependencies: acorn: 8.17.0 acorn@8.17.0: {} + agent-base@6.0.2: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + agent-base@7.1.4: {} ajv-draft-04@1.0.0(ajv@8.20.0): @@ -7928,7 +8237,7 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 - alchemy@0.93.12(@opentelemetry/api@1.9.0)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0))(workerd@1.20260617.1): + alchemy@0.93.12(@opentelemetry/api@1.9.1)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0))(workerd@1.20260617.1): dependencies: '@aws-sdk/credential-providers': 3.1073.0 '@cloudflare/unenv-preset': 2.7.7(unenv@2.0.0-rc.21)(workerd@1.20260617.1) @@ -7938,7 +8247,7 @@ snapshots: '@smithy/node-config-provider': 4.5.1 '@smithy/types': 4.15.0 aws4fetch: 1.0.20 - drizzle-orm: 0.45.2(@cloudflare/workers-types@4.20260621.1)(@opentelemetry/api@1.9.0) + drizzle-orm: 0.45.2(@cloudflare/workers-types@4.20260621.1)(@opentelemetry/api@1.9.1) env-paths: 3.0.0 esbuild: 0.25.12 execa: 9.6.1 @@ -8133,8 +8442,20 @@ snapshots: - uploadthing - yaml + asynckit@0.4.0: {} + aws4fetch@1.0.20: {} + axios@1.18.1: + dependencies: + follow-redirects: 1.16.0 + form-data: 4.0.6 + https-proxy-agent: 5.0.1 + proxy-from-env: 2.1.0 + transitivePeerDependencies: + - debug + - supports-color + axobject-query@4.1.0: {} bail@2.0.2: {} @@ -8244,6 +8565,8 @@ snapshots: ci-info@4.4.0: {} + cjs-module-lexer@2.2.0: {} + cli-cursor@5.0.0: dependencies: restore-cursor: 5.1.0 @@ -8269,6 +8592,10 @@ snapshots: color-name@1.1.4: {} + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + comma-separated-tokens@2.0.3: {} commander@11.1.0: {} @@ -8361,6 +8688,8 @@ snapshots: defu@6.1.7: {} + delayed-stream@1.0.0: {} + depd@2.0.0: {} dequal@2.0.3: {} @@ -8403,10 +8732,10 @@ snapshots: domelementtype: 2.3.0 domhandler: 5.0.3 - drizzle-orm@0.45.2(@cloudflare/workers-types@4.20260621.1)(@opentelemetry/api@1.9.0): + drizzle-orm@0.45.2(@cloudflare/workers-types@4.20260621.1)(@opentelemetry/api@1.9.1): optionalDependencies: '@cloudflare/workers-types': 4.20260621.1 - '@opentelemetry/api': 1.9.0 + '@opentelemetry/api': 1.9.1 dset@3.1.4: {} @@ -8480,6 +8809,13 @@ snapshots: dependencies: es-errors: 1.3.0 + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.4 + esast-util-from-estree@2.0.0: dependencies: '@types/estree-jsx': 1.0.5 @@ -8589,6 +8925,12 @@ snapshots: esprima@4.0.1: {} + esquery@1.7.0: + dependencies: + estraverse: 5.3.0 + + estraverse@5.3.0: {} + estree-util-attach-comments@3.0.0: dependencies: '@types/estree': 1.0.9 @@ -8802,6 +9144,8 @@ snapshots: flattie@1.1.1: {} + follow-redirects@1.16.0: {} + fontace@0.4.1: dependencies: fontkitten: 1.0.3 @@ -8815,6 +9159,14 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + form-data@4.0.6: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.4 + mime-types: 2.1.35 + formdata-node@6.0.3: {} formdata-polyfill@4.0.10: @@ -8954,6 +9306,10 @@ snapshots: has-symbols@1.1.0: {} + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + hasown@2.0.4: dependencies: function-bind: 1.1.2 @@ -9189,6 +9545,13 @@ snapshots: transitivePeerDependencies: - supports-color + https-proxy-agent@5.0.1: + dependencies: + agent-base: 6.0.2 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.4 @@ -9216,6 +9579,13 @@ snapshots: immediate@3.0.6: {} + import-in-the-middle@3.2.0: + dependencies: + acorn: 8.17.0 + acorn-import-attributes: 1.9.5(acorn@8.17.0) + cjs-module-lexer: 2.2.0 + module-details-from-path: 1.0.4 + inherits@2.0.4: {} ini@7.0.0: {} @@ -9664,6 +10034,8 @@ snapshots: merge2@1.4.1: {} + meriyah@6.1.4: {} + meshoptimizer@1.1.1: {} micromark-core-commonmark@2.0.3: @@ -9945,8 +10317,14 @@ snapshots: braces: 3.0.3 picomatch: 2.3.2 + mime-db@1.52.0: {} + mime-db@1.54.0: {} + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + mime-types@3.0.2: dependencies: mime-db: 1.54.0 @@ -9989,6 +10367,8 @@ snapshots: minipass@7.1.3: {} + module-details-from-path@1.0.4: {} + mri@1.2.0: {} mrmime@2.0.1: {} @@ -10290,6 +10670,13 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + posthog-node@4.18.0: + dependencies: + axios: 1.18.1 + transitivePeerDependencies: + - debug + - supports-color + prettier-plugin-astro@0.14.1: dependencies: '@astrojs/compiler': 2.13.1 @@ -10335,6 +10722,8 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 + proxy-from-env@2.1.0: {} + punycode@2.3.1: {} pure-rand@8.4.0: {} @@ -10537,6 +10926,13 @@ snapshots: require-from-string@2.0.2: {} + require-in-the-middle@8.0.1: + dependencies: + debug: 4.4.3 + module-details-from-path: 1.0.4 + transitivePeerDependencies: + - supports-color + resolve-from@5.0.0: {} resolve-pkg-maps@1.0.0: {} @@ -10661,6 +11057,8 @@ snapshots: sax@1.6.0: {} + semifies@1.0.0: {} + semver@7.8.0: {} semver@7.8.5: {} @@ -10803,6 +11201,8 @@ snapshots: source-map-js@1.2.1: {} + source-map@0.6.1: {} + source-map@0.7.6: {} space-separated-tokens@2.0.2: {} @@ -11171,6 +11571,34 @@ snapshots: transitivePeerDependencies: - msw + vitest@4.1.9(@opentelemetry/api@1.9.1)(@types/node@25.9.4)(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)): + dependencies: + '@vitest/expect': 4.1.9 + '@vitest/mocker': 4.1.9(vite@7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)) + '@vitest/pretty-format': 4.1.9 + '@vitest/runner': 4.1.9 + '@vitest/snapshot': 4.1.9 + '@vitest/spy': 4.1.9 + '@vitest/utils': 4.1.9 + es-module-lexer: 2.1.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.3 + pathe: 2.0.3 + picomatch: 4.0.4 + std-env: 4.1.0 + tinybench: 2.9.0 + tinyexec: 1.2.4 + tinyglobby: 0.2.17 + tinyrainbow: 3.1.0 + vite: 7.3.5(@types/node@25.9.4)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@types/node': 25.9.4 + transitivePeerDependencies: + - msw + volar-service-css@0.0.70(@volar/language-service@2.4.28): dependencies: vscode-css-languageservice: 6.3.10 diff --git a/schemas/caplets-config.schema.json b/schemas/caplets-config.schema.json index 90021fe3..46cbc33b 100644 --- a/schemas/caplets-config.schema.json +++ b/schemas/caplets-config.schema.json @@ -29,6 +29,10 @@ "exclusiveMinimum": 0, "maximum": 50 }, + "telemetry": { + "description": "Set false to disable anonymous Caplets telemetry for this user config.", + "type": "boolean" + }, "completion": { "default": { "discoveryTimeoutMs": 750, diff --git a/scripts/check-telemetry-release-env.ts b/scripts/check-telemetry-release-env.ts new file mode 100644 index 00000000..850edbbc --- /dev/null +++ b/scripts/check-telemetry-release-env.ts @@ -0,0 +1,79 @@ +const REQUIRED_TELEMETRY_RELEASE_ENV = [ + { + name: "CAPLETS_POSTHOG_TOKEN", + label: "PostHog project token", + validate: isNonPlaceholderSecret, + }, + { + name: "CAPLETS_SENTRY_DSN", + label: "Sentry DSN", + validate: isSentryDsn, + }, +] as const; + +export type TelemetryReleaseEnv = Record; + +export function checkTelemetryReleaseEnv(env: TelemetryReleaseEnv): string[] { + const failures: string[] = []; + + for (const required of REQUIRED_TELEMETRY_RELEASE_ENV) { + const value = env[required.name]; + if (value === undefined || value.trim() === "") { + failures.push(`${required.name} is required for telemetry-enabled releases.`); + continue; + } + if (!required.validate(value)) { + failures.push( + `${required.name} must be a valid ${required.label}; placeholders are not allowed.`, + ); + } + } + + return failures; +} + +function isNonPlaceholderSecret(value: string): boolean { + const normalized = value.trim().toLowerCase(); + if (normalized === "") return false; + return ![ + "todo", + "todo before release", + "changeme", + "change-me", + "placeholder", + "example", + ].includes(normalized); +} + +function isSentryDsn(value: string): boolean { + if (!isNonPlaceholderSecret(value)) return false; + + try { + const url = new URL(value); + return ( + (url.protocol === "https:" || url.protocol === "http:") && + url.username.length > 0 && + url.hostname.length > 0 && + url.pathname !== "" && + url.pathname !== "/" + ); + } catch { + return false; + } +} + +if (import.meta.url === new URL(process.argv[1] ?? "", "file:").href) { + const failures = checkTelemetryReleaseEnv(process.env); + if (failures.length > 0) { + console.error("Telemetry release environment is not configured:"); + for (const failure of failures) { + console.error(`- ${failure}`); + } + console.error( + "Configure GitHub Actions secrets CAPLETS_POSTHOG_TOKEN and CAPLETS_SENTRY_DSN before publishing.", + ); + process.exit(1); + } + + console.log("Telemetry release environment is configured."); +} From c7459db5d70f0fa6575e7e0c72ca6468c5cecee6 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Wed, 24 Jun 2026 08:40:55 -0400 Subject: [PATCH 2/6] fix: make telemetry notice test hermetic --- packages/core/test/telemetry-cli.test.ts | 39 +++++++++++++++++++++--- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/packages/core/test/telemetry-cli.test.ts b/packages/core/test/telemetry-cli.test.ts index e23dde43..00a951f3 100644 --- a/packages/core/test/telemetry-cli.test.ts +++ b/packages/core/test/telemetry-cli.test.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -14,6 +14,26 @@ function tempDir(): string { return dir; } +function writeMinimalConfig(path: string): void { + writeFileSync( + path, + `${JSON.stringify( + { + mcpServers: { + example: { + name: "Example", + description: "Example test server.", + command: "node", + args: ["server.js"], + }, + }, + }, + null, + 2, + )}\n`, + ); +} + describe("telemetry CLI", () => { afterEach(() => { for (const dir of dirs.splice(0)) { @@ -77,9 +97,15 @@ describe("telemetry CLI", () => { const dir = tempDir(); const out: string[] = []; const err: string[] = []; + const configPath = join(dir, "config.json"); + writeMinimalConfig(configPath); + const env = { + CAPLETS_CONFIG: configPath, + CAPLETS_PROJECT_CONFIG: join(dir, "project", ".caplets", "config.json"), + }; await runCli(["serve"], { - env: {}, + env, telemetryStateDir: join(dir, "state"), stderrIsTTY: true, writeOut: (value) => out.push(value), @@ -93,7 +119,7 @@ describe("telemetry CLI", () => { expect(readTelemetryNotice({ stateDir: join(dir, "state") }).shown).toBe(true); await runCli(["serve"], { - env: {}, + env, telemetryStateDir: join(dir, "state"), stderrIsTTY: true, writeErr: (value) => err.push(value), @@ -105,9 +131,14 @@ describe("telemetry CLI", () => { it("does not mark notice shown when stderr is redirected", async () => { const dir = tempDir(); const err: string[] = []; + const configPath = join(dir, "config.json"); + writeMinimalConfig(configPath); await runCli(["serve"], { - env: {}, + env: { + CAPLETS_CONFIG: configPath, + CAPLETS_PROJECT_CONFIG: join(dir, "project", ".caplets", "config.json"), + }, telemetryStateDir: join(dir, "state"), stderrIsTTY: false, writeErr: (value) => err.push(value), From e1ff2ea136145effc9519159d3c4b20cbe93d982 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Wed, 24 Jun 2026 09:21:49 -0400 Subject: [PATCH 3/6] fix: address telemetry review feedback --- .github/workflows/release.yml | 8 +- package.json | 1 + packages/core/src/cli.ts | 201 +++++------------- packages/core/src/cli/code-mode.ts | 15 +- packages/core/src/code-mode/runner.ts | 15 +- packages/core/src/code-mode/tool.ts | 1 + packages/core/src/code-mode/types.ts | 1 + packages/core/src/engine.ts | 28 ++- packages/core/src/native/service.ts | 77 ++++++- packages/core/src/telemetry/context.ts | 7 +- packages/core/src/telemetry/events.ts | 2 +- packages/core/src/telemetry/index.ts | 2 + .../core/src/telemetry/intake.generated.ts | 3 + packages/core/src/telemetry/notice.ts | 2 + packages/core/src/telemetry/providers.ts | 14 +- packages/core/src/telemetry/runtime.ts | 105 ++++++++- packages/core/src/telemetry/state.ts | 6 +- packages/core/test/code-mode-runner.test.ts | 1 + packages/core/test/telemetry-cli.test.ts | 36 ++++ packages/core/test/telemetry-events.test.ts | 13 ++ .../core/test/telemetry-providers.test.ts | 2 +- packages/core/test/telemetry-release.test.ts | 2 +- packages/core/test/telemetry-runtime.test.ts | 34 +++ packages/core/test/telemetry-state.test.ts | 35 +-- scripts/check-telemetry-release-env.ts | 23 ++ 25 files changed, 431 insertions(+), 203 deletions(-) create mode 100644 packages/core/src/telemetry/intake.generated.ts create mode 100644 packages/core/test/telemetry-runtime.test.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b396c41a..e91d27aa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,18 +51,12 @@ jobs: - name: Verify package run: pnpm verify - - name: Check telemetry release env - env: - CAPLETS_POSTHOG_TOKEN: ${{ secrets.CAPLETS_POSTHOG_TOKEN }} - CAPLETS_SENTRY_DSN: ${{ secrets.CAPLETS_SENTRY_DSN }} - run: pnpm telemetry:check-release-env - - name: Create release PR or publish id: changesets uses: changesets/action@v1 with: version: pnpm version-packages - publish: pnpm release + publish: pnpm telemetry:prepare-release-env && pnpm release commit: "chore: version packages" title: "chore: version packages" env: diff --git a/package.json b/package.json index 88947de2..e1550cc0 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "schema:generate": "tsx ./scripts/generate-config-schema.ts", "test": "vitest run", "telemetry:check-release-env": "tsx ./scripts/check-telemetry-release-env.ts", + "telemetry:prepare-release-env": "tsx ./scripts/check-telemetry-release-env.ts --write-bundled-intake", "typecheck": "tsgo --noEmit && turbo typecheck", "verify": "pnpm format:check && pnpm lint && pnpm code-mode:check-api && pnpm schema:check && pnpm docs:check && pnpm typecheck && pnpm test && pnpm benchmark:check && pnpm build", "version-packages": "changeset version && oxlint --fix --quiet && oxfmt --write ." diff --git a/packages/core/src/cli.ts b/packages/core/src/cli.ts index c8074d44..14b5e867 100644 --- a/packages/core/src/cli.ts +++ b/packages/core/src/cli.ts @@ -376,9 +376,9 @@ function telemetryCommandFamilyFromArgs( if (command === cliCommands.remote || command === cliCommands.cloud) { return { commandFamily: "remote", surface: "cli" }; } + if (command === cliCommands.inspect) return { commandFamily: "inspect", surface: "cli" }; + if (command === cliCommands.checkBackend) return { commandFamily: "check", surface: "cli" }; if ( - command === cliCommands.inspect || - command === cliCommands.checkBackend || command === cliCommands.listTools || command === cliCommands.searchTools || command === cliCommands.getTool || @@ -428,6 +428,8 @@ function maybePrintCliTelemetryNotice( stateDir: context.stateDir, surface, visibility: "visible", + allowWithoutNotice: true, + createIdentity: false, }); if (state.status !== "enabled" || state.notice.shown) return; maybePrintTelemetryNotice({ @@ -451,9 +453,7 @@ async function captureCliTelemetry( productEvent?: boolean | undefined; }, ): Promise { - if (options.productEvent !== false) { - maybePrintCliTelemetryNotice(context, options.surface ?? "cli"); - } + maybePrintCliTelemetryNotice(context, options.surface ?? "cli"); const state = resolveTelemetryState({ config: telemetryConfigForCli(context), env: context.env, @@ -1311,6 +1311,17 @@ export function createProgram(io: CliIO = {}): Command { ((code: number) => { process.exitCode = code; }); + const executeOperationIo = (format: CliOutputFormat | undefined): ExecuteOperationIO => ({ + writeOut, + writeErr, + setExitCode, + authDir: io.authDir, + env, + remote: remoteClientForCli(io), + format, + telemetryStateDir: telemetryContext().stateDir, + telemetryDebugSink: io.telemetryDebugSink, + }); const program = new Command(); program @@ -1449,16 +1460,19 @@ export function createProgram(io: CliIO = {}): Command { .action(async (args: string[]) => { const nestedArgs = args[0] === "--" ? args.slice(1) : args; const sink = new TelemetryDebugSink(); - if (nestedArgs.length > 0) { - await runCli(nestedArgs, { - ...io, - env: { ...env, CAPLETS_TELEMETRY_DEBUG: "1" }, - telemetryDebugSink: sink, - writeOut, - writeErr, - }); + try { + if (nestedArgs.length > 0) { + await runCli(nestedArgs, { + ...io, + env: { ...env, CAPLETS_TELEMETRY_DEBUG: "1" }, + telemetryDebugSink: sink, + writeOut, + writeErr, + }); + } + } finally { + writeOut(`${JSON.stringify({ telemetryDebug: sink.toJSON() }, null, 2)}\n`); } - writeOut(`${JSON.stringify({ telemetryDebug: sink.toJSON() }, null, 2)}\n`); }); const codeMode = program @@ -1514,6 +1528,7 @@ export function createProgram(io: CliIO = {}): Command { ...(currentConfigPath() ? { configPath: currentConfigPath() } : {}), projectConfigPath: envProjectConfigPath(env), ...(io.authDir ? { authDir: io.authDir } : {}), + telemetryStateDir: telemetryContext().stateDir, ...(code === undefined ? {} : { inlineCode: code }), ...(options.file === undefined ? {} : { file: options.file }), ...(options.sessionId === undefined ? {} : { sessionId: options.sessionId }), @@ -1537,6 +1552,7 @@ export function createProgram(io: CliIO = {}): Command { ...(currentConfigPath() ? { configPath: currentConfigPath() } : {}), projectConfigPath: envProjectConfigPath(env), ...(io.authDir ? { authDir: io.authDir } : {}), + telemetryStateDir: telemetryContext().stateDir, ...(options.json === undefined && parentOptions.json === undefined ? {} : { json: options.json ?? parentOptions.json }), @@ -3132,19 +3148,7 @@ export function createProgram(io: CliIO = {}): Command { .argument("", "configured Caplet ID") .option("--format ", "output format: markdown, md, plain, or json", parseOutputFormat) .action(async (caplet: string, options: { format?: CliOutputFormat }) => { - await executeOperation( - caplet, - { operation: "inspect" }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, - ); + await executeOperation(caplet, { operation: "inspect" }, executeOperationIo(options.format)); }); program @@ -3153,19 +3157,7 @@ export function createProgram(io: CliIO = {}): Command { .argument("", "configured Caplet ID") .option("--format ", "output format: markdown, md, plain, or json", parseOutputFormat) .action(async (caplet: string, options: { format?: CliOutputFormat }) => { - await executeOperation( - caplet, - { operation: "check" }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, - ); + await executeOperation(caplet, { operation: "check" }, executeOperationIo(options.format)); }); program @@ -3174,19 +3166,7 @@ export function createProgram(io: CliIO = {}): Command { .argument("", "configured Caplet ID") .option("--format ", "output format: markdown, md, plain, or json", parseOutputFormat) .action(async (caplet: string, options: { format?: CliOutputFormat }) => { - await executeOperation( - caplet, - { operation: "tools" }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, - ); + await executeOperation(caplet, { operation: "tools" }, executeOperationIo(options.format)); }); program @@ -3207,15 +3187,7 @@ export function createProgram(io: CliIO = {}): Command { options.limit === undefined ? { operation: "search_tools", query } : { operation: "search_tools", query, limit: options.limit }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, + executeOperationIo(options.format), ); }, ); @@ -3236,15 +3208,7 @@ export function createProgram(io: CliIO = {}): Command { await executeOperation( caplet, { operation: "describe_tool", name: tool }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, + executeOperationIo(options.format), ); }, ); @@ -3270,15 +3234,7 @@ export function createProgram(io: CliIO = {}): Command { args: parseCallToolArgs(options.args), ...(options.field && options.field.length > 0 ? { fields: options.field } : {}), }; - await executeOperation(caplet, request, { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }); + await executeOperation(caplet, request, executeOperationIo(options.format)); }, ); @@ -3294,15 +3250,7 @@ export function createProgram(io: CliIO = {}): Command { options.limit === undefined ? { operation: "resources" } : { operation: "resources", limit: options.limit }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, + executeOperationIo(options.format), ), ); program @@ -3323,15 +3271,7 @@ export function createProgram(io: CliIO = {}): Command { options.limit === undefined ? { operation: "search_resources", query } : { operation: "search_resources", query, limit: options.limit }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, + executeOperationIo(options.format), ), ); program @@ -3346,15 +3286,7 @@ export function createProgram(io: CliIO = {}): Command { options.limit === undefined ? { operation: "resource_templates" } : { operation: "resource_templates", limit: options.limit }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, + executeOperationIo(options.format), ), ); program @@ -3367,15 +3299,7 @@ export function createProgram(io: CliIO = {}): Command { executeOperation( caplet, { operation: "read_resource", uri }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, + executeOperationIo(options.format), ), ); program @@ -3390,15 +3314,7 @@ export function createProgram(io: CliIO = {}): Command { options.limit === undefined ? { operation: "prompts" } : { operation: "prompts", limit: options.limit }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, + executeOperationIo(options.format), ), ); program @@ -3419,15 +3335,7 @@ export function createProgram(io: CliIO = {}): Command { options.limit === undefined ? { operation: "search_prompts", query } : { operation: "search_prompts", query, limit: options.limit }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, + executeOperationIo(options.format), ), ); program @@ -3451,15 +3359,7 @@ export function createProgram(io: CliIO = {}): Command { name: prompt, args: parseJsonObjectOption(options.args, "get-prompt --args"), }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, + executeOperationIo(options.format), ); }, ); @@ -3490,15 +3390,7 @@ export function createProgram(io: CliIO = {}): Command { ref: completionRefFromOptions(options), argument: { name: options.argument, value: options.value }, }, - { - writeOut, - writeErr, - setExitCode, - authDir: io.authDir, - env, - remote: remoteClientForCli(io), - format: options.format, - }, + executeOperationIo(options.format), ), ); @@ -4326,6 +4218,8 @@ type ExecuteOperationIO = Required; remote?: RemoteControlClient | undefined; format?: CliOutputFormat | undefined; + telemetryStateDir?: string | undefined; + telemetryDebugSink?: TelemetryDebugSink | undefined; }; type CapletListRow = ReturnType[number]; @@ -4584,10 +4478,11 @@ async function executeLocalOperation( watch: false, writeErr: io.writeErr, telemetryEnv: io.env ?? process.env, - telemetryStateDir: defaultTelemetryStateDir(io.env ?? process.env), + telemetryStateDir: io.telemetryStateDir ?? defaultTelemetryStateDir(io.env ?? process.env), telemetrySurface: "cli", telemetryVisibility: "visible", telemetryRuntimeMode: runtimeModeForEnv(io.env ?? process.env), + telemetryDebugSink: io.telemetryDebugSink, ...(config ? { configLoader: () => config } : {}), }); try { diff --git a/packages/core/src/cli/code-mode.ts b/packages/core/src/cli/code-mode.ts index ac3d147a..86cf5662 100644 --- a/packages/core/src/cli/code-mode.ts +++ b/packages/core/src/cli/code-mode.ts @@ -18,6 +18,7 @@ export type CodeModeCliOptions = { configPath?: string | undefined; projectConfigPath?: string | undefined; authDir?: string | undefined; + telemetryStateDir?: string | undefined; inlineCode?: string | undefined; file?: string | undefined; timeoutMs?: number | undefined; @@ -36,7 +37,7 @@ export async function runCodeModeCli(options: CodeModeCliOptions): Promise ...(options.projectConfigPath ? { projectConfigPath: options.projectConfigPath } : {}), ...(options.authDir ? { authDir: options.authDir } : {}), telemetryEnv: options.env as NodeJS.ProcessEnv | undefined, - telemetryStateDir: defaultTelemetryStateDir(options.env), + telemetryStateDir: options.telemetryStateDir ?? defaultTelemetryStateDir(options.env), telemetrySurface: "code_mode", telemetryVisibility: "visible", telemetryRuntimeMode: runtimeScope(options.env) === "local" ? "local" : "unknown", @@ -70,7 +71,7 @@ export async function runCodeModeCli(options: CodeModeCliOptions): Promise service: service.codeModeService?.() ?? service, runtimeScope: "cli-one-shot", }); - void service + await service .captureCodeModeOutcome?.(result, { started, ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), @@ -103,6 +104,7 @@ export async function runCodeModeReplCli( | "configPath" | "projectConfigPath" | "authDir" + | "telemetryStateDir" | "sessionId" | "recoveryRef" | "json" @@ -132,13 +134,20 @@ export async function runCodeModeReplCli( export async function codeModeTypesCli( options: Pick< CodeModeCliOptions, - "env" | "configPath" | "projectConfigPath" | "authDir" | "json" | "writeOut" + | "env" + | "configPath" + | "projectConfigPath" + | "authDir" + | "telemetryStateDir" + | "json" + | "writeOut" >, ): Promise { const engine = new CapletsEngine({ ...(options.configPath ? { configPath: options.configPath } : {}), ...(options.projectConfigPath ? { projectConfigPath: options.projectConfigPath } : {}), ...(options.authDir ? { authDir: options.authDir } : {}), + telemetryStateDir: options.telemetryStateDir ?? defaultTelemetryStateDir(options.env), }); try { const caplets = listCodeModeCallableCaplets(engine); diff --git a/packages/core/src/code-mode/runner.ts b/packages/core/src/code-mode/runner.ts index ca10fbd0..cb6fb4db 100644 --- a/packages/core/src/code-mode/runner.ts +++ b/packages/core/src/code-mode/runner.ts @@ -43,6 +43,7 @@ export async function runCodeMode(input: RunCodeModeInput): Promise = { + const metaBase: Omit = { runId: randomUUID(), traceId: randomUUID(), declarationHash, @@ -63,7 +64,11 @@ export async function runCodeMode(input: RunCodeModeInput): Promise ({ ...metaBase, durationMs: Date.now() - startedAt }); + const meta = (): CodeModeRunMeta => ({ + ...metaBase, + durationMs: Date.now() - startedAt, + anyCapletInvoked: invokedCaplet, + }); if (input.sessionId !== undefined && !input.sessionManager) { return { @@ -170,7 +175,6 @@ export async function runCodeMode(input: RunCodeModeInput): Promise input.logStore?.read(readInput) ?? { entries: [] }, @@ -378,7 +382,10 @@ async function journalRun( } } -function setRecoveryMeta(metaBase: Omit, recoveryRef: string): void { +function setRecoveryMeta( + metaBase: Omit, + recoveryRef: string, +): void { metaBase.recoveryRef = recoveryRef; } diff --git a/packages/core/src/code-mode/tool.ts b/packages/core/src/code-mode/tool.ts index 85489746..ee6734f4 100644 --- a/packages/core/src/code-mode/tool.ts +++ b/packages/core/src/code-mode/tool.ts @@ -53,6 +53,7 @@ export function emptyCodeModeRunMeta(): CodeModeRunMeta { durationMs: 0, timeoutMs: 0, maxTimeoutMs: 0, + anyCapletInvoked: false, sessionId: null, sessionStatus: null, recoveryRef: null, diff --git a/packages/core/src/code-mode/types.ts b/packages/core/src/code-mode/types.ts index 19622a56..805686ab 100644 --- a/packages/core/src/code-mode/types.ts +++ b/packages/core/src/code-mode/types.ts @@ -39,6 +39,7 @@ export type CodeModeRunMeta = { durationMs: number; timeoutMs: number; maxTimeoutMs: number; + anyCapletInvoked: boolean; sessionId?: string | null; sessionStatus?: CodeModeSessionStatus | null; recoveryRef?: string | null; diff --git a/packages/core/src/engine.ts b/packages/core/src/engine.ts index f04a26c3..12d1b038 100644 --- a/packages/core/src/engine.ts +++ b/packages/core/src/engine.ts @@ -29,9 +29,11 @@ import { ServerRegistry } from "./registry"; import { handleServerTool } from "./tools"; import { discoverExposureSnapshot, type ExposureSnapshot } from "./exposure/discovery"; import { + captureRuntimeReliabilityEvent, captureRuntimeTelemetryEvent, codeModeTelemetryProperties, createRuntimeTelemetryContext, + runtimeFailureTelemetryProperties, toolActivationProperties, type RuntimeMode, type RuntimeTelemetryContext, @@ -108,6 +110,7 @@ export class CapletsEngine { private readonly observedOutputShapeScope: ObservedOutputShapeKey["scope"]; private readonly projectFingerprint: string | undefined; private readonly telemetry: RuntimeTelemetryContext; + private readonly telemetryExecuteExposureMode: "progressive" | "code_mode"; private readonly reloadListeners = new Set<(event: CapletsEngineReloadEvent) => void>(); private lastExposureSnapshot: ExposureSnapshot | undefined; private watchers: FSWatcher[] = []; @@ -138,6 +141,8 @@ export class CapletsEngine { debugSink: options.telemetryDebugSink, dispatcher: options.telemetryDispatcher, }); + this.telemetryExecuteExposureMode = + options.telemetrySurface === "code_mode" ? "code_mode" : "progressive"; this.downstream = new DownstreamManager(this.registry, selectAuthOptions(options.authDir)); this.openapi = new OpenApiManager(this.registry, selectHttpLikeOptions(options)); this.googleDiscovery = new GoogleDiscoveryManager( @@ -257,17 +262,22 @@ export class CapletsEngine { this.captureToolActivation( caplet, operationFromRequest(request), - "progressive", + this.telemetryExecuteExposureMode, result, started, ); return result; } catch (error) { const result = errorResult(error); + this.captureReliabilityError( + operationFromRequest(request), + this.telemetryExecuteExposureMode, + result, + ); this.captureToolActivation( caplet, operationFromRequest(request), - "progressive", + this.telemetryExecuteExposureMode, result, started, ); @@ -290,6 +300,7 @@ export class CapletsEngine { return annotated; } catch (error) { const result = errorResult(error); + this.captureReliabilityError("call_tool", "direct", result); this.captureToolActivation(caplet, "call_tool", "direct", result, started); return result; } @@ -307,6 +318,7 @@ export class CapletsEngine { return annotated; } catch (error) { const result = errorResult(error); + this.captureReliabilityError("read_resource", "direct", result); this.captureToolActivation(caplet, "read_resource", "direct", result, started); return result; } @@ -328,6 +340,7 @@ export class CapletsEngine { return annotated; } catch (error) { const result = errorResult(error); + this.captureReliabilityError("get_prompt", "direct", result); this.captureToolActivation(caplet, "get_prompt", "direct", result, started); return result; } @@ -662,6 +675,17 @@ export class CapletsEngine { }, this.watchDebounceMs); } + private captureReliabilityError( + operation: unknown, + exposureMode: "direct" | "progressive" | "code_mode" | "mixed" | "unknown", + result: unknown, + ): void { + void captureRuntimeReliabilityEvent(this.telemetry, { + command_family: commandFamilyForTelemetrySurface(this.telemetry.surface), + ...runtimeFailureTelemetryProperties({ operation, exposureMode, result }), + }).catch(() => undefined); + } + private captureToolActivation( caplet: CapletConfig | undefined, operation: unknown, diff --git a/packages/core/src/native/service.ts b/packages/core/src/native/service.ts index 25bd25c0..fdc960e2 100644 --- a/packages/core/src/native/service.ts +++ b/packages/core/src/native/service.ts @@ -18,7 +18,7 @@ import { type SdkRemoteCapletsClientOptions, } from "./remote"; import { CapletsEngine } from "../engine"; -import { CapletsError } from "../errors"; +import { CapletsError, errorResult } from "../errors"; import type { RuntimeMode, TelemetryDebugSink, @@ -26,6 +26,14 @@ import type { TelemetrySurface, TelemetryVisibility, } from "../telemetry"; +import { + captureRuntimeReliabilityEvent, + captureRuntimeTelemetryEvent, + createRuntimeTelemetryContext, + runtimeFailureTelemetryProperties, + toolActivationProperties, + type RuntimeTelemetryContext, +} from "../telemetry"; import { nativeCapletPromptGuidance, nativeCapletToolDescription, @@ -562,11 +570,16 @@ function nativeMcpPrimitiveRequest( return { operation: operationName }; } +function operationFromNativeRequest(request: unknown): unknown { + return isRecord(request) ? request.operation : undefined; +} + function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } function runtimeModeFromNativeOptions(options: NativeCapletsServiceOptions) { + if (options.mode === "local") return "local"; if (options.mode === "remote") return "remote"; if (options.mode === "cloud") return "cloud"; if (options.remote?.url) return "remote"; @@ -575,6 +588,13 @@ function runtimeModeFromNativeOptions(options: NativeCapletsServiceOptions) { return "local"; } +function telemetryConfigFromNativeOptions(options: NativeCapletsServiceOptions): CapletsConfig { + return createLocalOverlayConfigLoader(options)( + resolveConfigPath(options.configPath), + options.projectConfigPath ?? resolveProjectConfigPath(), + ); +} + function codeModeRunNativeTool(capletTools: NativeCapletTool[]): NativeCapletTool { const codeModeCaplets = capletTools.map((tool) => ({ id: tool.caplet, @@ -1012,6 +1032,7 @@ class CompositeNativeCapletsService implements NativeCapletsService { private closed = false; private batchingReload = false; private readonly codeModeSessions = new CodeModeSessionManager(); + private readonly telemetry: RuntimeTelemetryContext; constructor( private remote: NativeCapletsService, @@ -1022,6 +1043,17 @@ class CompositeNativeCapletsService implements NativeCapletsService { ) { this.unsubscribeRemote = this.remote.onToolsChanged(() => this.updateMergedTools()); this.unsubscribeLocal = this.local.onToolsChanged(() => this.updateMergedTools()); + this.telemetry = createRuntimeTelemetryContext({ + config: telemetryConfigFromNativeOptions(options), + env: options.telemetryEnv, + stateDir: options.telemetryStateDir, + surface: options.telemetrySurface ?? "native", + visibility: options.telemetryVisibility ?? "hidden", + runtimeMode: options.telemetryRuntimeMode ?? runtimeModeFromNativeOptions(options), + integration: options.telemetryIntegration ?? "native", + debugSink: options.telemetryDebugSink, + dispatcher: options.telemetryDispatcher, + }); const merged = this.mergeTools(); this.tools = merged.tools; this.routes = merged.routes; @@ -1042,13 +1074,13 @@ class CompositeNativeCapletsService implements NativeCapletsService { return await this.local.execute(route.capletId, request); } if (route?.service === "remote") { - return await this.remote.execute(route.capletId, request); + return await this.executeRemote(route.capletId, request); } const diagnostic = this.namespaceDiagnostics.get(capletId); if (diagnostic) { throw new CapletsError("CAPLET_NAMESPACE_COLLISION", diagnostic.hint, diagnostic); } - return await this.remote.execute(capletId, request); + return await this.executeRemote(capletId, request); } codeModeService(): NativeCapletsService { @@ -1148,6 +1180,45 @@ class CompositeNativeCapletsService implements NativeCapletsService { } } + private async executeRemote(capletId: string, request: unknown): Promise { + const started = Date.now(); + try { + const result = await this.remote.execute(capletId, request); + this.captureRemoteToolActivation(request, result, started); + return result; + } catch (error) { + const result = errorResult(error); + this.captureRemoteReliabilityError(request, result); + this.captureRemoteToolActivation(request, result, started); + throw error; + } + } + + private captureRemoteToolActivation(request: unknown, result: unknown, started: number): void { + void captureRuntimeTelemetryEvent(this.telemetry, "caplets_tool_activation", { + command_family: "native", + ...toolActivationProperties({ + config: this.telemetry.config, + caplet: undefined, + operation: operationFromNativeRequest(request), + exposureMode: "direct", + result, + durationMs: Date.now() - started, + }), + }).catch(() => undefined); + } + + private captureRemoteReliabilityError(request: unknown, result: unknown): void { + void captureRuntimeReliabilityEvent(this.telemetry, { + command_family: "native", + ...runtimeFailureTelemetryProperties({ + operation: operationFromNativeRequest(request), + exposureMode: "direct", + result, + }), + }).catch(() => undefined); + } + private mergeTools(): { tools: NativeCapletTool[]; routes: Map; diff --git a/packages/core/src/telemetry/context.ts b/packages/core/src/telemetry/context.ts index d5f30f15..6f001b68 100644 --- a/packages/core/src/telemetry/context.ts +++ b/packages/core/src/telemetry/context.ts @@ -44,12 +44,7 @@ export function resolveTelemetryState(options: ResolveTelemetryStateOptions): Te return { ...base, status: "disabled", decider: "test" }; } - if ( - executionContext !== "ci" && - !notice.shown && - !options.allowWithoutNotice && - options.visibility !== "visible" - ) { + if (executionContext !== "ci" && !notice.shown && !options.allowWithoutNotice) { return { ...base, status: "suppressed", decider: "notice" }; } diff --git a/packages/core/src/telemetry/events.ts b/packages/core/src/telemetry/events.ts index beb178d1..139243ee 100644 --- a/packages/core/src/telemetry/events.ts +++ b/packages/core/src/telemetry/events.ts @@ -124,8 +124,8 @@ export function buildProductTelemetryEvent(input: { name: input.name, distinctId: input.distinctId, properties: { - $process_person_profile: false, ...input.properties, + $process_person_profile: false, }, }; } diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts index 40265eea..e9a92b45 100644 --- a/packages/core/src/telemetry/index.ts +++ b/packages/core/src/telemetry/index.ts @@ -52,12 +52,14 @@ export { } from "./providers"; export { backendFamilyCounts, + captureRuntimeReliabilityEvent, captureRuntimeTelemetryEvent, codeModeTelemetryProperties, createRuntimeTelemetryContext, exposureModeCounts, operationFamilyFromOperation, outcomeFromResult, + runtimeFailureTelemetryProperties, toolActivationProperties, type RuntimeTelemetryContext, type RuntimeTelemetryOptions, diff --git a/packages/core/src/telemetry/intake.generated.ts b/packages/core/src/telemetry/intake.generated.ts new file mode 100644 index 00000000..99871bd7 --- /dev/null +++ b/packages/core/src/telemetry/intake.generated.ts @@ -0,0 +1,3 @@ +// Release builds overwrite this file in CI before package build/publish. +export const BUNDLED_POSTHOG_TOKEN = undefined as string | undefined; +export const BUNDLED_SENTRY_DSN = undefined as string | undefined; diff --git a/packages/core/src/telemetry/notice.ts b/packages/core/src/telemetry/notice.ts index 5ef40ca9..e47b486c 100644 --- a/packages/core/src/telemetry/notice.ts +++ b/packages/core/src/telemetry/notice.ts @@ -1,4 +1,5 @@ import { + readTelemetryNotice, recordTelemetryNoticeShown, type TelemetrySurface, type TelemetryStateOptions, @@ -15,6 +16,7 @@ export type TelemetryNoticeOptions = TelemetryStateOptions & { export function maybePrintTelemetryNotice(options: TelemetryNoticeOptions): boolean { if (!options.stderrIsTTY) return false; + if (readTelemetryNotice(options).shown) return false; options.writeErr(TELEMETRY_NOTICE); recordTelemetryNoticeShown({ ...options, surface: options.surface }); return true; diff --git a/packages/core/src/telemetry/providers.ts b/packages/core/src/telemetry/providers.ts index 8398b08f..b1710704 100644 --- a/packages/core/src/telemetry/providers.ts +++ b/packages/core/src/telemetry/providers.ts @@ -1,6 +1,7 @@ import type { NodeClient, NodeOptions } from "@sentry/node"; import type { PostHog } from "posthog-node"; import type { ProductTelemetryEvent, ReliabilityTelemetryEvent, TelemetryEvent } from "./events"; +import { BUNDLED_POSTHOG_TOKEN, BUNDLED_SENTRY_DSN } from "./intake.generated"; import { stripSentryEvent } from "./privacy"; import { recordTelemetryDrop, type TelemetryState } from "./state"; @@ -31,14 +32,15 @@ export function createTelemetryDispatcher( let sentry: Promise | undefined; async function posthogClient(): Promise { - const token = options.posthogToken ?? process.env.CAPLETS_POSTHOG_TOKEN; + const token = + options.posthogToken ?? process.env.CAPLETS_POSTHOG_TOKEN ?? BUNDLED_POSTHOG_TOKEN; if (!token) return undefined; posthog ??= Promise.resolve((options.factories?.createPostHog ?? defaultPostHogFactory)(token)); return posthog; } async function sentryClient(): Promise { - const dsn = options.sentryDsn ?? process.env.CAPLETS_SENTRY_DSN; + const dsn = options.sentryDsn ?? process.env.CAPLETS_SENTRY_DSN ?? BUNDLED_SENTRY_DSN; if (!dsn) return undefined; sentry ??= Promise.resolve((options.factories?.createSentry ?? defaultSentryFactory)(dsn)); return sentry; @@ -91,8 +93,8 @@ async function capturePostHog( distinctId: event.distinctId, event: event.name, properties: { - $geoip_disable: true, ...event.properties, + $geoip_disable: true, }, }); } @@ -129,13 +131,15 @@ async function defaultSentryFactory(dsn: string): Promise { dsn, sendDefaultPii: false, defaultIntegrations: false, + integrations: [], tracesSampleRate: 0, + transport: sentry.makeNodeTransport, + stackParser: sentry.defaultStackParser, beforeSend(event) { return stripSentryEvent( event as unknown as Record, ) as unknown as typeof event; }, } satisfies NodeOptions; - sentry.init(options); - return sentry.getClient() as NodeClient; + return new sentry.NodeClient(options); } diff --git a/packages/core/src/telemetry/runtime.ts b/packages/core/src/telemetry/runtime.ts index 8604c79f..6fa7d204 100644 --- a/packages/core/src/telemetry/runtime.ts +++ b/packages/core/src/telemetry/runtime.ts @@ -1,8 +1,11 @@ +import { arch, platform } from "node:os"; import { version as packageJsonVersion } from "../../package.json"; import type { CapletConfig, CapletsConfig } from "../config"; import { resolveExposure } from "../exposure/policy"; import { + buildReliabilityTelemetryEvent, buildProductTelemetryEvent, + type DiagnosticCategory, durationBucket, timeoutBucket, type CommandFamily, @@ -79,6 +82,43 @@ export async function captureRuntimeTelemetryEvent( await context.dispatcher.capture(state, event); } +export async function captureRuntimeReliabilityEvent( + context: RuntimeTelemetryContext, + properties: TelemetryProperties, +): Promise { + const state = resolveTelemetryState({ + config: context.config, + env: context.env, + stateDir: context.stateDir, + surface: context.surface, + visibility: context.visibility, + debug: context.debugSink !== undefined, + }); + if (state.status !== "enabled" && state.status !== "debug") { + return; + } + const event = buildReliabilityTelemetryEvent({ + name: "caplets_reliability_error", + properties: { + package: "@caplets/core", + version: packageJsonVersion, + surface: context.surface, + runtime_mode: context.runtimeMode ?? "unknown", + execution_context: state.executionContext, + ...(context.integration ? { integration: context.integration } : {}), + os_family: platform(), + arch: arch(), + node_major: Number(process.versions.node.split(".")[0] ?? 0), + ...properties, + }, + }); + if (state.status === "debug") { + context.debugSink?.capture("debug", event); + return; + } + await context.dispatcher.capture(state, event); +} + export function backendFamilyCounts(config: CapletsConfig): TelemetryProperties { return { backend_mcp_count: enabledCount(config.mcpServers), @@ -116,6 +156,7 @@ export function operationFamilyFromOperation(operation: unknown): CommandFamily operation === "tools" || operation === "search_tools" || operation === "get_tool" || + operation === "describe_tool" || operation === "call_tool" ) { return "tools"; @@ -124,12 +165,19 @@ export function operationFamilyFromOperation(operation: unknown): CommandFamily operation === "resources" || operation === "resource_templates" || operation === "read_resource" || + operation === "search_resources" || operation === "list_resources" || - operation === "list_resource_templates" + operation === "list_resource_templates" || + operation === "search_resource_templates" ) { return "resources"; } - if (operation === "prompts" || operation === "get_prompt" || operation === "list_prompts") { + if ( + operation === "prompts" || + operation === "get_prompt" || + operation === "list_prompts" || + operation === "search_prompts" + ) { return "prompts"; } if (operation === "complete") return "complete"; @@ -141,7 +189,7 @@ export function outcomeFromResult(result: unknown): Outcome { if (isRecord(result) && result.isError === true) return "failure"; if (isRecord(result) && result.ok === false) { const code = isRecord(result.error) ? result.error.code : undefined; - if (code === "TIMEOUT") return "timeout"; + if (typeof code === "string" && code.toLowerCase().includes("timeout")) return "timeout"; return "failure"; } return "success"; @@ -162,7 +210,7 @@ export function codeModeTelemetryProperties( timeout_bucket: timeoutBucket(timeoutMs), session_category: sessionStatus === "created" || sessionStatus === "reused" ? sessionStatus : "unknown", - any_caplet_invoked: false, + any_caplet_invoked: codeModeEnvelopeInvokedCaplet(record), }; } @@ -203,3 +251,52 @@ function allCaplets(config: CapletsConfig): CapletConfig[] { function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } + +export function runtimeFailureTelemetryProperties(input: { + operation: unknown; + exposureMode: "direct" | "progressive" | "code_mode" | "mixed" | "unknown"; + result: unknown; +}): TelemetryProperties { + const errorCode = errorCodeFromResult(input.result); + return { + operation_family: operationFamilyFromOperation(input.operation), + exposure_mode: input.exposureMode, + error_code: errorCode, + diagnostic_category: diagnosticCategoryFromCode(errorCode), + }; +} + +function errorCodeFromResult(result: unknown): string { + if (!isRecord(result)) return "UNKNOWN"; + const error = result.error; + if (isRecord(error) && typeof error.code === "string") return error.code; + return "UNKNOWN"; +} + +function diagnosticCategoryFromCode(code: string): DiagnosticCategory { + if (code.startsWith("CONFIG")) return "config"; + if (code.startsWith("AUTH")) return "auth"; + if (code.includes("NETWORK") || code.includes("UNAVAILABLE")) return "network"; + if (code.includes("VALID") || code.includes("REQUEST")) return "validation"; + if (code.includes("CODE_MODE") || code.includes("SANDBOX")) return "code_mode"; + return "runtime"; +} + +function codeModeEnvelopeInvokedCaplet(record: Record): boolean { + const meta = isRecord(record.meta) ? record.meta : undefined; + return ( + hasBooleanTrue(meta, "anyCapletInvoked") || + hasBooleanTrue(meta, "capletInvoked") || + hasPositiveNumber(meta, "capletInvocationCount") || + hasPositiveNumber(meta, "toolCallCount") + ); +} + +function hasBooleanTrue(record: Record | undefined, key: string): boolean { + return record?.[key] === true; +} + +function hasPositiveNumber(record: Record | undefined, key: string): boolean { + const value = record?.[key]; + return typeof value === "number" && value > 0; +} diff --git a/packages/core/src/telemetry/state.ts b/packages/core/src/telemetry/state.ts index 4c7983ab..30036137 100644 --- a/packages/core/src/telemetry/state.ts +++ b/packages/core/src/telemetry/state.ts @@ -79,7 +79,11 @@ export function rotateTelemetryIdentity(options: TelemetryStateOptions = {}): Te } export function deleteTelemetryIdentity(options: TelemetryStateOptions = {}): void { - rmSync(telemetryIdentityPath(options), { force: true }); + try { + rmSync(telemetryIdentityPath(options), { force: true }); + } catch { + // Telemetry state operations are best effort. + } } export function readTelemetryNotice(options: TelemetryStateOptions = {}): TelemetryNoticeState { diff --git a/packages/core/test/code-mode-runner.test.ts b/packages/core/test/code-mode-runner.test.ts index f18289f0..fdc3aede 100644 --- a/packages/core/test/code-mode-runner.test.ts +++ b/packages/core/test/code-mode-runner.test.ts @@ -146,6 +146,7 @@ describe("runCodeMode", () => { }); expect(result.ok).toBe(true); + expect(result.meta.anyCapletInvoked).toBe(true); expect(native.execute).toHaveBeenCalledWith("github", { operation: "call_tool", name: "listIssues", diff --git a/packages/core/test/telemetry-cli.test.ts b/packages/core/test/telemetry-cli.test.ts index 00a951f3..b1cadf09 100644 --- a/packages/core/test/telemetry-cli.test.ts +++ b/packages/core/test/telemetry-cli.test.ts @@ -217,6 +217,42 @@ describe("telemetry CLI", () => { }); }); + it("prints the first-run notice before parse-error reliability capture", async () => { + const dir = tempDir(); + const err: string[] = []; + const configPath = join(dir, "config.json"); + writeMinimalConfig(configPath); + + await expect( + runCli(["init", "--typo"], { + env: { CAPLETS_CONFIG: configPath }, + telemetryStateDir: join(dir, "state"), + stderrIsTTY: true, + writeErr: (value) => err.push(value), + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + + expect(err.join("")).toContain("Caplets collects anonymous telemetry"); + expect(readTelemetryNotice({ stateDir: join(dir, "state") }).shown).toBe(true); + }); + + it("prints telemetry debug output even when the nested command fails", async () => { + const dir = tempDir(); + const out: string[] = []; + + await expect( + runCli(["telemetry", "debug", "--", "init", "--typo"], { + telemetryStateDir: join(dir, "state"), + writeOut: (value) => out.push(value), + writeErr: () => {}, + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + + expect(out.join("")).toContain('"telemetryDebug"'); + expect(out.join("")).toContain('"provider": "sentry"'); + expect(out.join("")).toContain('"command_family": "init"'); + }); + it("completion includes telemetry subcommands", async () => { await expect(completeCliWords(["telemetry", ""])).resolves.toEqual([ "status", diff --git a/packages/core/test/telemetry-events.test.ts b/packages/core/test/telemetry-events.test.ts index 55325d23..d8bb5e15 100644 --- a/packages/core/test/telemetry-events.test.ts +++ b/packages/core/test/telemetry-events.test.ts @@ -41,6 +41,19 @@ describe("telemetry event builders", () => { }); }); + it("forces anonymous product profile behavior after caller properties", () => { + const event = buildProductTelemetryEvent({ + name: "caplets_cli_command", + distinctId: "anon_1234567890abcdef1234567890abcdef", + properties: { + surface: "cli", + $process_person_profile: true, + } as never, + }); + + expect(event.properties.$process_person_profile).toBe(false); + }); + it("rejects unknown event and property keys", () => { expect(() => buildProductTelemetryEvent({ diff --git a/packages/core/test/telemetry-providers.test.ts b/packages/core/test/telemetry-providers.test.ts index e41b207a..eebcab23 100644 --- a/packages/core/test/telemetry-providers.test.ts +++ b/packages/core/test/telemetry-providers.test.ts @@ -92,7 +92,7 @@ describe("telemetry providers", () => { buildProductTelemetryEvent({ name: "caplets_cli_command", distinctId: state.identity!.id, - properties: { surface: "cli", execution_context: "ci" }, + properties: { surface: "cli", execution_context: "ci", $geoip_disable: false } as never, }), ); diff --git a/packages/core/test/telemetry-release.test.ts b/packages/core/test/telemetry-release.test.ts index 16c31566..8b951f81 100644 --- a/packages/core/test/telemetry-release.test.ts +++ b/packages/core/test/telemetry-release.test.ts @@ -39,7 +39,7 @@ describe("telemetry release environment", () => { it("wires telemetry secrets into the release workflow", () => { const workflow = read(".github/workflows/release.yml"); - expect(workflow).toContain("pnpm telemetry:check-release-env"); + expect(workflow).toContain("publish: pnpm telemetry:prepare-release-env && pnpm release"); expect(workflow).toContain("CAPLETS_POSTHOG_TOKEN: ${{ secrets.CAPLETS_POSTHOG_TOKEN }}"); expect(workflow).toContain("CAPLETS_SENTRY_DSN: ${{ secrets.CAPLETS_SENTRY_DSN }}"); }); diff --git a/packages/core/test/telemetry-runtime.test.ts b/packages/core/test/telemetry-runtime.test.ts new file mode 100644 index 00000000..e682bafc --- /dev/null +++ b/packages/core/test/telemetry-runtime.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { + codeModeTelemetryProperties, + operationFamilyFromOperation, + outcomeFromResult, +} from "../src/telemetry"; + +describe("telemetry runtime helpers", () => { + it("maps current runtime operation names into stable families", () => { + expect(operationFamilyFromOperation("describe_tool")).toBe("tools"); + expect(operationFamilyFromOperation("search_resources")).toBe("resources"); + expect(operationFamilyFromOperation("search_prompts")).toBe("prompts"); + }); + + it("classifies timeout-shaped error codes as timeouts", () => { + expect(outcomeFromResult({ ok: false, error: { code: "sandbox_timeout" } })).toBe("timeout"); + }); + + it("reports Code Mode caplet invocation from run metadata", () => { + expect( + codeModeTelemetryProperties( + { + ok: true, + meta: { sessionStatus: "created", anyCapletInvoked: true }, + }, + 50, + 1_000, + ), + ).toMatchObject({ + command_family: "code_mode", + any_caplet_invoked: true, + }); + }); +}); diff --git a/packages/core/test/telemetry-state.test.ts b/packages/core/test/telemetry-state.test.ts index f15459bb..c8a54966 100644 --- a/packages/core/test/telemetry-state.test.ts +++ b/packages/core/test/telemetry-state.test.ts @@ -1,4 +1,12 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -108,16 +116,18 @@ describe("telemetry state", () => { expect(resolveTelemetryState({ ...base, env: { VITEST: "true" } }).decider).toBe("test"); }); - it("suppresses hidden native and daemon surfaces until notice exists", () => { + it("suppresses non-CI telemetry until notice exists", () => { const stateDir = tempRoot(); - const beforeNotice = resolveTelemetryState({ - stateDir, - env: {}, - surface: "native", - visibility: "hidden", - }); - expect(beforeNotice.status).toBe("suppressed"); - expect(beforeNotice.decider).toBe("notice"); + for (const visibility of ["visible", "hidden", "unknown"] as const) { + const beforeNotice = resolveTelemetryState({ + stateDir, + env: {}, + surface: "native", + visibility, + }); + expect(beforeNotice.status).toBe("suppressed"); + expect(beforeNotice.decider).toBe("notice"); + } recordTelemetryNoticeShown({ stateDir, surface: "cli" }); @@ -143,8 +153,9 @@ describe("telemetry state", () => { expect(state.executionContext).toBe("ci"); expect(readTelemetryNotice({ stateDir }).shown).toBe(false); - const readonlyState = join("/dev/null", "caplets-telemetry-state"); - const identity = readTelemetryIdentity({ stateDir: readonlyState, create: true }); + const blockedStateDir = join(tempRoot(), "blocked-state"); + writeFileSync(blockedStateDir, "not a directory"); + const identity = readTelemetryIdentity({ stateDir: blockedStateDir, create: true }); expect(identity.kind).toBe("ephemeral"); }); diff --git a/scripts/check-telemetry-release-env.ts b/scripts/check-telemetry-release-env.ts index 850edbbc..b549aed5 100644 --- a/scripts/check-telemetry-release-env.ts +++ b/scripts/check-telemetry-release-env.ts @@ -1,3 +1,5 @@ +import { writeFileSync } from "node:fs"; + const REQUIRED_TELEMETRY_RELEASE_ENV = [ { name: "CAPLETS_POSTHOG_TOKEN", @@ -11,6 +13,8 @@ const REQUIRED_TELEMETRY_RELEASE_ENV = [ }, ] as const; +const BUNDLED_INTAKE_PATH = "packages/core/src/telemetry/intake.generated.ts"; + export type TelemetryReleaseEnv = Record; export function checkTelemetryReleaseEnv(env: TelemetryReleaseEnv): string[] { @@ -32,6 +36,20 @@ export function checkTelemetryReleaseEnv(env: TelemetryReleaseEnv): string[] { return failures; } +export function writeBundledTelemetryIntake(env: TelemetryReleaseEnv): void { + const posthogToken = env.CAPLETS_POSTHOG_TOKEN; + const sentryDsn = env.CAPLETS_SENTRY_DSN; + writeFileSync( + BUNDLED_INTAKE_PATH, + [ + "// Generated by scripts/check-telemetry-release-env.ts during release builds.", + `export const BUNDLED_POSTHOG_TOKEN = ${JSON.stringify(posthogToken)} as string | undefined;`, + `export const BUNDLED_SENTRY_DSN = ${JSON.stringify(sentryDsn)} as string | undefined;`, + "", + ].join("\n"), + ); +} + function isNonPlaceholderSecret(value: string): boolean { const normalized = value.trim().toLowerCase(); if (normalized === "") return false; @@ -75,5 +93,10 @@ if (import.meta.url === new URL(process.argv[1] ?? "", "file:").href) { process.exit(1); } + if (process.argv.includes("--write-bundled-intake")) { + writeBundledTelemetryIntake(process.env); + console.log(`Wrote bundled telemetry intake identifiers to ${BUNDLED_INTAKE_PATH}.`); + } + console.log("Telemetry release environment is configured."); } From f939655ca79272ce55c566db29e909394f639d5d Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Wed, 24 Jun 2026 09:43:31 -0400 Subject: [PATCH 4/6] fix: keep telemetry from masking cli errors --- packages/core/src/cli.ts | 2 +- packages/core/test/telemetry-cli.test.ts | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/core/src/cli.ts b/packages/core/src/cli.ts index 14b5e867..9a333ea7 100644 --- a/packages/core/src/cli.ts +++ b/packages/core/src/cli.ts @@ -229,7 +229,7 @@ export async function runCli(args: string[], io: CliIO = {}): Promise { startedAt, error: normalizedError, productEvent: captureProductEvent, - }); + }).catch(() => undefined); } throw normalizedError; } finally { diff --git a/packages/core/test/telemetry-cli.test.ts b/packages/core/test/telemetry-cli.test.ts index b1cadf09..6e238d2f 100644 --- a/packages/core/test/telemetry-cli.test.ts +++ b/packages/core/test/telemetry-cli.test.ts @@ -236,6 +236,26 @@ describe("telemetry CLI", () => { expect(readTelemetryNotice({ stateDir: join(dir, "state") }).shown).toBe(true); }); + it("does not let parse-error telemetry failures mask the command error", async () => { + const dir = tempDir(); + const configPath = join(dir, "config.json"); + let writeErrCalls = 0; + writeMinimalConfig(configPath); + + await expect( + runCli(["init", "--typo"], { + env: { CAPLETS_CONFIG: configPath }, + telemetryStateDir: join(dir, "state"), + stderrIsTTY: true, + writeErr: () => { + writeErrCalls += 1; + if (writeErrCalls === 1) return; + throw new Error("telemetry notice write failed"); + }, + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + }); + it("prints telemetry debug output even when the nested command fails", async () => { const dir = tempDir(); const out: string[] = []; From 6cb9d7fa8413c448fd0f43dc14b17a00b50eba4a Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Wed, 24 Jun 2026 10:02:43 -0400 Subject: [PATCH 5/6] fix: flush composite native telemetry --- packages/core/src/native/service.ts | 10 ++++++- packages/core/test/native-remote.test.ts | 33 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/packages/core/src/native/service.ts b/packages/core/src/native/service.ts index fdc960e2..37215ec1 100644 --- a/packages/core/src/native/service.ts +++ b/packages/core/src/native/service.ts @@ -1033,6 +1033,7 @@ class CompositeNativeCapletsService implements NativeCapletsService { private batchingReload = false; private readonly codeModeSessions = new CodeModeSessionManager(); private readonly telemetry: RuntimeTelemetryContext; + private readonly ownsTelemetryDispatcher: boolean; constructor( private remote: NativeCapletsService, @@ -1043,6 +1044,7 @@ class CompositeNativeCapletsService implements NativeCapletsService { ) { this.unsubscribeRemote = this.remote.onToolsChanged(() => this.updateMergedTools()); this.unsubscribeLocal = this.local.onToolsChanged(() => this.updateMergedTools()); + this.ownsTelemetryDispatcher = options.telemetryDispatcher === undefined; this.telemetry = createRuntimeTelemetryContext({ config: telemetryConfigFromNativeOptions(options), env: options.telemetryEnv, @@ -1110,6 +1112,7 @@ class CompositeNativeCapletsService implements NativeCapletsService { this.local.listTools().map((tool) => tool.caplet), ); } + this.telemetry.config = telemetryConfigFromNativeOptions(this.options); this.startPresence(); this.updateMergedTools(); return remoteReloaded || localReloaded; @@ -1129,7 +1132,12 @@ class CompositeNativeCapletsService implements NativeCapletsService { this.unsubscribeLocal(); this.listeners.clear(); this.codeModeSessions.close(); - await Promise.all([this.remote.close(), this.local.close(), this.presence?.close()]); + await Promise.all([ + this.remote.close(), + this.local.close(), + this.presence?.close(), + this.ownsTelemetryDispatcher ? this.telemetry.dispatcher.shutdown() : undefined, + ]); } async replaceRemote( diff --git a/packages/core/test/native-remote.test.ts b/packages/core/test/native-remote.test.ts index 7266a9f5..96944a60 100644 --- a/packages/core/test/native-remote.test.ts +++ b/packages/core/test/native-remote.test.ts @@ -19,6 +19,7 @@ import { type NativeCapletsService, resetNativeProjectBindingFallbackWarningForTests, } from "../src/native/service"; +import { recordTelemetryNoticeShown } from "../src/telemetry"; import { FileRemoteProfileStore } from "../src/remote/profile-store"; import { createHttpServeApp } from "../src/serve/http"; import type { HttpServeOptions } from "../src/serve/options"; @@ -1718,6 +1719,38 @@ describe("createNativeCapletsService remote mode", () => { await service.close(); }); + it("refreshes composite telemetry config on reload and shuts down the dispatcher", async () => { + const fixture = client([{ name: "alpha", title: "Alpha", description: "Remote alpha" }]); + const { dir, configPath, projectConfigPath } = tempConfig({}); + dirs.push(dir); + const stateDir = join(dir, "state"); + recordTelemetryNoticeShown({ stateDir, surface: "cli" }); + const capture = vi.fn(); + const shutdown = vi.fn(async () => undefined); + const service = createNativeCapletsService({ + mode: "remote", + remote: { url: "http://127.0.0.1:5387" }, + remoteClientFactory: vi.fn(() => fixture.api), + configPath, + projectConfigPath, + telemetryStateDir: stateDir, + telemetryEnv: {}, + telemetryDispatcher: { capture, shutdown }, + }); + + await service.reload(); + await service.execute("alpha", { operation: "inspect" }); + await expect.poll(() => capture.mock.calls.length).toBe(1); + + writeFileSync(configPath, JSON.stringify(progressiveTestConfig({ telemetry: false })), "utf8"); + await expect(service.reload()).resolves.toBe(true); + await service.execute("alpha", { operation: "inspect" }); + await expect.poll(() => capture.mock.calls.length).toBe(1); + + await service.close(); + expect(shutdown).toHaveBeenCalledTimes(1); + }); + it("loads self-hosted native remote credentials from a saved Remote Profile", async () => { const authDir = mkdtempSync(join(tmpdir(), "caplets-native-remote-auth-")); dirs.push(authDir); From 2c89f57ceaa41b118e1caf46ae7d0d2dd65c4344 Mon Sep 17 00:00:00 2001 From: Ian Pascoe Date: Wed, 24 Jun 2026 10:11:14 -0400 Subject: [PATCH 6/6] fix: harden telemetry review edge cases --- docs/product/telemetry-readout.md | 25 +++++++------ packages/core/src/cli.ts | 32 ++++++++++++++--- packages/core/src/native/service.ts | 23 ++++++++++-- packages/core/src/telemetry/events.ts | 8 +---- packages/core/src/telemetry/runtime.ts | 24 ++++++++++--- packages/core/test/native-remote.test.ts | 35 ++++++++++++++++++ packages/core/test/telemetry-cli.test.ts | 37 ++++++++++++++++++++ packages/core/test/telemetry-docs.test.ts | 2 -- packages/core/test/telemetry-runtime.test.ts | 36 +++++++++++++++++++ 9 files changed, 188 insertions(+), 34 deletions(-) diff --git a/docs/product/telemetry-readout.md b/docs/product/telemetry-readout.md index 4b2ef9b7..fb99d5bd 100644 --- a/docs/product/telemetry-readout.md +++ b/docs/product/telemetry-readout.md @@ -4,22 +4,21 @@ This readout maps anonymous event families back to the product questions that mo ## Decision Questions -| Question | Event families | Required properties | -| --------------------------------------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Where does setup fail? | `caplets_setup_milestone`, `caplets_cli_command`, `caplets_reliability_error` | `command_family`, `outcome`, `duration_bucket`, `error_code`, `diagnostic_category` | -| Which surfaces are active? | `caplets_cli_command`, `caplets_runtime_lifecycle`, `caplets_tool_activation` | `surface`, `execution_context`, `runtime_mode`, `outcome` | -| Is local, remote, or cloud runtime worth more investment? | `caplets_runtime_lifecycle`, `caplets_tool_activation`, `caplets_reliability_error` | `runtime_mode`, `surface`, `outcome`, `error_code` | -| Are native integrations used? | `caplets_runtime_lifecycle`, `caplets_tool_activation`, `caplets_code_mode_outcome` | `surface`, `integration`, `runtime_mode`, `outcome` | -| Which exposure modes are used? | `caplets_tool_activation` | `exposure_mode`, `direct_count`, `progressive_count`, `code_mode_count`, `operation_family` | -| Which backend families deserve investment? | `caplets_tool_activation`, `caplets_runtime_lifecycle` | `backend_mcp_count`, `backend_openapi_count`, `backend_google_discovery_count`, `backend_graphql_count`, `backend_http_count`, `backend_cli_count`, `backend_caplets_count` | -| Is Code Mode succeeding? | `caplets_code_mode_outcome`, `caplets_reliability_error` | `outcome`, `duration_bucket`, `timeout_bucket`, `session_category`, `any_caplet_invoked`, `diagnostic_category` | -| What reliability pressure is highest? | `caplets_reliability_error`, `caplets_delivery_health` | `surface`, `runtime_mode`, `command_family`, `error_code`, `diagnostic_category`, `provider`, `reason`, `count_bucket` | +| Question | Event families | Required properties | +| --------------------------------------------------------- | ----------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Where does setup fail? | `caplets_cli_command`, `caplets_reliability_error` | `command_family`, `outcome`, `duration_bucket`, `error_code`, `diagnostic_category` | +| Which surfaces are active? | `caplets_cli_command`, `caplets_tool_activation`, `caplets_code_mode_outcome` | `surface`, `execution_context`, `runtime_mode`, `outcome` | +| Is local, remote, or cloud runtime worth more investment? | `caplets_cli_command`, `caplets_tool_activation`, `caplets_reliability_error` | `runtime_mode`, `surface`, `outcome`, `error_code` | +| Are native integrations used? | `caplets_tool_activation`, `caplets_code_mode_outcome` | `surface`, `integration`, `runtime_mode`, `outcome` | +| Which exposure modes are used? | `caplets_tool_activation` | `exposure_mode`, `direct_count`, `progressive_count`, `code_mode_count`, `operation_family` | +| Which backend families deserve investment? | `caplets_tool_activation` | `backend_mcp_count`, `backend_openapi_count`, `backend_google_discovery_count`, `backend_graphql_count`, `backend_http_count`, `backend_cli_count`, `backend_caplets_count` | +| Is Code Mode succeeding? | `caplets_code_mode_outcome`, `caplets_reliability_error` | `outcome`, `duration_bucket`, `timeout_bucket`, `session_category`, `any_caplet_invoked`, `diagnostic_category` | +| What reliability pressure is highest? | `caplets_reliability_error` | `surface`, `runtime_mode`, `command_family`, `error_code`, `diagnostic_category` | ## Saved Query Contract -- Setup funnel: count `caplets_setup_milestone` by `command_family` and `outcome`. -- Surface adoption: count `caplets_runtime_lifecycle` by `surface`, `runtime_mode`, and `execution_context`. +- Setup funnel: count `caplets_cli_command` setup/install outcomes by `command_family` and `outcome`. +- Surface adoption: count `caplets_cli_command`, `caplets_tool_activation`, and `caplets_code_mode_outcome` by `surface`, `runtime_mode`, and `execution_context`. - First activation: count `caplets_tool_activation` successes by `surface`, `operation_family`, and `exposure_mode`. - Code Mode outcomes: count `caplets_code_mode_outcome` by `outcome`, `timeout_bucket`, and `session_category`. - Reliability: count `caplets_reliability_error` by `surface`, `command_family`, `error_code`, and `diagnostic_category`. -- Delivery health: count `caplets_delivery_health` by `provider`, `reason`, and `count_bucket`. diff --git a/packages/core/src/cli.ts b/packages/core/src/cli.ts index 9a333ea7..c3b335d7 100644 --- a/packages/core/src/cli.ts +++ b/packages/core/src/cli.ts @@ -414,7 +414,16 @@ function telemetryConfigForCli(context: TelemetryCliContext): Pick | undefined { + try { + const config = readUserConfigObject(path); + return typeof config.telemetry === "boolean" ? { telemetry: config.telemetry } : undefined; + } catch { + return undefined; } } @@ -453,7 +462,11 @@ async function captureCliTelemetry( productEvent?: boolean | undefined; }, ): Promise { - maybePrintCliTelemetryNotice(context, options.surface ?? "cli"); + try { + maybePrintCliTelemetryNotice(context, options.surface ?? "cli"); + } catch { + // Telemetry notice delivery is best-effort and must not affect command behavior. + } const state = resolveTelemetryState({ config: telemetryConfigForCli(context), env: context.env, @@ -492,6 +505,7 @@ async function captureCliTelemetry( } if (options.outcome !== "failure") return; + if (options.error === undefined) return; const reliability = buildReliabilityTelemetryEvent({ name: "caplets_reliability_error", properties: { @@ -1304,8 +1318,13 @@ export function createProgram(io: CliIO = {}): Command { stderrIsTTY: io.stderrIsTTY ?? process.stderr.isTTY === true, writeErr, }); - const printTelemetryNotice = (surface: TelemetrySurface) => - maybePrintCliTelemetryNotice(telemetryContext(), surface); + const printTelemetryNotice = (surface: TelemetrySurface) => { + try { + maybePrintCliTelemetryNotice(telemetryContext(), surface); + } catch { + // Telemetry notice delivery is best-effort and must not affect command behavior. + } + }; const setExitCode = io.setExitCode ?? ((code: number) => { @@ -1495,6 +1514,11 @@ export function createProgram(io: CliIO = {}): Command { json?: boolean; }, ) => { + try { + maybePrintCliTelemetryNotice(telemetryContext(), "code_mode"); + } catch { + // Telemetry notice delivery is best-effort and must not affect Code Mode. + } if (code === "repl" && options.file === undefined) { await runCodeModeReplCli({ env, diff --git a/packages/core/src/native/service.ts b/packages/core/src/native/service.ts index 37215ec1..19f19f52 100644 --- a/packages/core/src/native/service.ts +++ b/packages/core/src/native/service.ts @@ -1,4 +1,4 @@ -import { realpathSync } from "node:fs"; +import { existsSync, readFileSync, realpathSync } from "node:fs"; import { resolve as resolvePath } from "node:path"; import { isLoopbackHost } from "../server/options"; import type { NativeCapletsServiceResolutionInput } from "./options"; @@ -589,10 +589,27 @@ function runtimeModeFromNativeOptions(options: NativeCapletsServiceOptions) { } function telemetryConfigFromNativeOptions(options: NativeCapletsServiceOptions): CapletsConfig { - return createLocalOverlayConfigLoader(options)( - resolveConfigPath(options.configPath), + const configPath = resolveConfigPath(options.configPath); + const config = createLocalOverlayConfigLoader(options)( + configPath, options.projectConfigPath ?? resolveProjectConfigPath(), ); + const explicitTelemetry = readTelemetryOnlyConfig(configPath); + return explicitTelemetry === undefined ? config : { ...config, telemetry: explicitTelemetry }; +} + +function readTelemetryOnlyConfig(path: string): boolean | undefined { + if (!existsSync(path)) return undefined; + try { + const parsed = JSON.parse(readFileSync(path, "utf8")); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? typeof (parsed as Record).telemetry === "boolean" + ? ((parsed as Record).telemetry as boolean) + : undefined + : undefined; + } catch { + return undefined; + } } function codeModeRunNativeTool(capletTools: NativeCapletTool[]): NativeCapletTool { diff --git a/packages/core/src/telemetry/events.ts b/packages/core/src/telemetry/events.ts index 139243ee..fabfefc4 100644 --- a/packages/core/src/telemetry/events.ts +++ b/packages/core/src/telemetry/events.ts @@ -38,11 +38,8 @@ export type DiagnosticCategory = export type TelemetryProductEventName = | "caplets_cli_command" - | "caplets_setup_milestone" - | "caplets_runtime_lifecycle" | "caplets_tool_activation" - | "caplets_code_mode_outcome" - | "caplets_delivery_health"; + | "caplets_code_mode_outcome"; export type TelemetryReliabilityEventName = "caplets_reliability_error"; @@ -101,11 +98,8 @@ export type TelemetryEvent = ProductTelemetryEvent | ReliabilityTelemetryEvent; const PRODUCT_EVENTS = new Set([ "caplets_cli_command", - "caplets_setup_milestone", - "caplets_runtime_lifecycle", "caplets_tool_activation", "caplets_code_mode_outcome", - "caplets_delivery_health", ]); const RELIABILITY_EVENTS = new Set(["caplets_reliability_error"]); diff --git a/packages/core/src/telemetry/runtime.ts b/packages/core/src/telemetry/runtime.ts index 6fa7d204..a9484a1c 100644 --- a/packages/core/src/telemetry/runtime.ts +++ b/packages/core/src/telemetry/runtime.ts @@ -186,9 +186,13 @@ export function operationFamilyFromOperation(operation: unknown): CommandFamily } export function outcomeFromResult(result: unknown): Outcome { - if (isRecord(result) && result.isError === true) return "failure"; + if (isRecord(result) && result.isError === true) { + const code = errorCodeFromResult(result); + if (code.toLowerCase().includes("timeout")) return "timeout"; + return "failure"; + } if (isRecord(result) && result.ok === false) { - const code = isRecord(result.error) ? result.error.code : undefined; + const code = errorCodeFromResult(result); if (typeof code === "string" && code.toLowerCase().includes("timeout")) return "timeout"; return "failure"; } @@ -203,13 +207,19 @@ export function codeModeTelemetryProperties( const record = isRecord(envelope) ? envelope : {}; const meta = isRecord(record.meta) ? record.meta : {}; const sessionStatus = meta.sessionStatus; + const effectiveTimeoutMs = + timeoutMs ?? (typeof meta.timeoutMs === "number" ? meta.timeoutMs : undefined); return { command_family: "code_mode", outcome: outcomeFromResult(record), duration_bucket: durationBucket(durationMs), - timeout_bucket: timeoutBucket(timeoutMs), + timeout_bucket: timeoutBucket(effectiveTimeoutMs), session_category: - sessionStatus === "created" || sessionStatus === "reused" ? sessionStatus : "unknown", + sessionStatus === "created" || sessionStatus === "reused" + ? sessionStatus + : sessionStatus === null + ? "none" + : "unknown", any_caplet_invoked: codeModeEnvelopeInvokedCaplet(record), }; } @@ -268,7 +278,11 @@ export function runtimeFailureTelemetryProperties(input: { function errorCodeFromResult(result: unknown): string { if (!isRecord(result)) return "UNKNOWN"; - const error = result.error; + const error = isRecord(result.error) + ? result.error + : isRecord(result.structuredContent) && isRecord(result.structuredContent.error) + ? result.structuredContent.error + : undefined; if (isRecord(error) && typeof error.code === "string") return error.code; return "UNKNOWN"; } diff --git a/packages/core/test/native-remote.test.ts b/packages/core/test/native-remote.test.ts index 96944a60..c44d6aed 100644 --- a/packages/core/test/native-remote.test.ts +++ b/packages/core/test/native-remote.test.ts @@ -1751,6 +1751,41 @@ describe("createNativeCapletsService remote mode", () => { expect(shutdown).toHaveBeenCalledTimes(1); }); + it("honors native telemetry opt-out when local overlay config has invalid backends", async () => { + const fixture = client([{ name: "alpha", title: "Alpha", description: "Remote alpha" }]); + const { dir, configPath, projectConfigPath } = tempConfig({}); + dirs.push(dir); + writeFileSync( + configPath, + JSON.stringify({ + telemetry: false, + mcpServers: { + invalid: { name: "Invalid", description: "Missing command." }, + }, + }), + "utf8", + ); + const stateDir = join(dir, "state"); + recordTelemetryNoticeShown({ stateDir, surface: "cli" }); + const capture = vi.fn(); + const service = createNativeCapletsService({ + mode: "remote", + remote: { url: "http://127.0.0.1:5387" }, + remoteClientFactory: vi.fn(() => fixture.api), + configPath, + projectConfigPath, + telemetryStateDir: stateDir, + telemetryEnv: {}, + telemetryDispatcher: { capture, shutdown: vi.fn(async () => undefined) }, + }); + + await service.reload(); + await service.execute("alpha", { operation: "inspect" }); + + expect(capture).not.toHaveBeenCalled(); + await service.close(); + }); + it("loads self-hosted native remote credentials from a saved Remote Profile", async () => { const authDir = mkdtempSync(join(tmpdir(), "caplets-native-remote-auth-")); dirs.push(authDir); diff --git a/packages/core/test/telemetry-cli.test.ts b/packages/core/test/telemetry-cli.test.ts index 6e238d2f..9b5ffa91 100644 --- a/packages/core/test/telemetry-cli.test.ts +++ b/packages/core/test/telemetry-cli.test.ts @@ -72,6 +72,25 @@ describe("telemetry CLI", () => { expect(JSON.parse(readFileSync(userConfig, "utf8")).telemetry).toBe(true); }); + it("preserves telemetry-only enablement when no backends are configured", async () => { + const dir = tempDir(); + const configPath = join(dir, "config.json"); + const out: string[] = []; + + await runCli(["telemetry", "enable"], { + env: { CAPLETS_CONFIG: configPath }, + writeOut: () => {}, + }); + await runCli(["telemetry", "status"], { + env: { CAPLETS_CONFIG: configPath, CI: "true" }, + telemetryStateDir: join(dir, "state"), + writeOut: (value) => out.push(value), + }); + + expect(out.join("")).toContain("Telemetry: enabled"); + expect(out.join("")).toContain("Config: enabled"); + }); + it("delete-id and rotate-id manage only local identity state", async () => { const dir = tempDir(); const stateDir = join(dir, "state"); @@ -193,6 +212,24 @@ describe("telemetry CLI", () => { expect(readTelemetryNotice({ stateDir: join(dir, "state") }).shown).toBe(true); }); + it("does not let telemetry notice write failures break successful commands", async () => { + const dir = tempDir(); + const configPath = join(dir, "config.json"); + writeMinimalConfig(configPath); + + await expect( + runCli(["serve"], { + env: { CAPLETS_CONFIG: configPath }, + telemetryStateDir: join(dir, "state"), + stderrIsTTY: true, + writeErr: () => { + throw new Error("telemetry notice write failed"); + }, + serve: async () => {}, + }), + ).resolves.toBeUndefined(); + }); + it("captures sanitized reliability but no product event for parse errors in debug mode", async () => { const dir = tempDir(); const sink = new TelemetryDebugSink(); diff --git a/packages/core/test/telemetry-docs.test.ts b/packages/core/test/telemetry-docs.test.ts index 3b5c56f9..71341626 100644 --- a/packages/core/test/telemetry-docs.test.ts +++ b/packages/core/test/telemetry-docs.test.ts @@ -41,11 +41,9 @@ describe("telemetry product docs", () => { "Is Code Mode succeeding?", "What reliability pressure is highest?", "caplets_cli_command", - "caplets_runtime_lifecycle", "caplets_tool_activation", "caplets_code_mode_outcome", "caplets_reliability_error", - "caplets_delivery_health", ]) { expect(text).toContain(expected); } diff --git a/packages/core/test/telemetry-runtime.test.ts b/packages/core/test/telemetry-runtime.test.ts index e682bafc..5c2547d0 100644 --- a/packages/core/test/telemetry-runtime.test.ts +++ b/packages/core/test/telemetry-runtime.test.ts @@ -3,6 +3,7 @@ import { codeModeTelemetryProperties, operationFamilyFromOperation, outcomeFromResult, + runtimeFailureTelemetryProperties, } from "../src/telemetry"; describe("telemetry runtime helpers", () => { @@ -14,6 +15,12 @@ describe("telemetry runtime helpers", () => { it("classifies timeout-shaped error codes as timeouts", () => { expect(outcomeFromResult({ ok: false, error: { code: "sandbox_timeout" } })).toBe("timeout"); + expect( + outcomeFromResult({ + isError: true, + structuredContent: { error: { code: "OPERATION_TIMEOUT" } }, + }), + ).toBe("timeout"); }); it("reports Code Mode caplet invocation from run metadata", () => { @@ -31,4 +38,33 @@ describe("telemetry runtime helpers", () => { any_caplet_invoked: true, }); }); + + it("uses Code Mode envelope timeout and one-shot session metadata", () => { + expect( + codeModeTelemetryProperties( + { + ok: true, + meta: { sessionStatus: null, timeoutMs: 10_000 }, + }, + 50, + undefined, + ), + ).toMatchObject({ + timeout_bucket: "lt_60s", + session_category: "none", + }); + }); + + it("reads reliability codes from structured MCP error content", () => { + expect( + runtimeFailureTelemetryProperties({ + operation: "call_tool", + exposureMode: "progressive", + result: { + isError: true, + structuredContent: { error: { code: "REQUEST_INVALID" } }, + }, + }), + ).toMatchObject({ error_code: "REQUEST_INVALID", diagnostic_category: "validation" }); + }); });