fix(config): quarantine missing env var caplets

.changeset/quiet-ravens-sneeze.md

@@ -0,0 +1,5 @@
+---
+"@caplets/core": patch
+---
+
+Keep local overlay startup alive when a Caplet references a missing environment variable by skipping only the affected Caplet and warning with the missing variable and config path.

.gitignore

@@ -33,5 +33,8 @@ benchmark-results/
 # autoresearch
 .auto/
 
-# Compound Engineering local config
+# compound-engineering
 .compound-engineering/*.local.yaml
+
+# worktrees
+.worktrees/

packages/core/src/config.ts

@@ -359,6 +359,7 @@ export type LocalOverlayConfigWarning = {
   kind: ConfigSourceKind;
   path: string;
   message: string;
+  recoverable?: boolean | undefined;
 };
 
 export type LocalOverlayConfigWithSources = ConfigWithSources & {
@@ -1074,6 +1075,23 @@ type ConfigInput = {
   [key: string]: unknown;
 };
 
+const CAPLET_BACKEND_KEYS = [
+  "mcpServers",
+  "openapiEndpoints",
+  "googleDiscoveryApis",
+  "graphqlEndpoints",
+  "httpApis",
+  "cliTools",
+  "capletSets",
+] as const satisfies ReadonlyArray<keyof ConfigInput>;
+
+const CAPLET_BACKEND_KEY_SET = new Set<string>(CAPLET_BACKEND_KEYS);
+
+type MissingEnvReference = {
+  name: string;
+  path: string;
+};
+
 function configSchemaFor(
   serverValueSchema: z.ZodTypeAny,
   openApiEndpointValueSchema: z.ZodTypeAny,
@@ -1712,14 +1730,107 @@ function readBestEffortConfigInput(
   transform?: (input: ConfigInput) => ConfigInput,
 ): ConfigInput | undefined {
   try {
-    const input = readPublicConfigInput(path);
-    return transform ? transform(input) : input;
+    const input = readBestEffortJsonConfigInput(path);
+    const normalized = normalizeLocalPaths(input, dirname(path));
+    const transformed = transform ? transform(normalized) : normalized;
+    const filtered = quarantineMissingEnvCaplets(transformed, kind, path, warnings);
+    const parsed = configFileSchema.safeParse(interpolateConfig(filtered));
+    if (!parsed.success) {
+      throw new CapletsError(
+        "CONFIG_INVALID",
+        `Caplets config at ${path} is invalid`,
+        parsed.error.issues,
+      );
+    }
+    return filtered;
   } catch (error) {
     warnings.push({ kind, path, message: errorMessage(error) });
     return undefined;
   }
 }
 
+function readBestEffortJsonConfigInput(path: string): ConfigInput {
+  try {
+    return JSON.parse(readFileSync(path, "utf8")) as ConfigInput;
+  } catch (error) {
+    throw new CapletsError(
+      "CONFIG_INVALID",
+      `Caplets config at ${path} is not valid JSON`,
+      redactSecrets(error),
+    );
+  }
+}
+
+function quarantineMissingEnvCaplets(
+  input: ConfigInput,
+  kind: ConfigSourceKind,
+  sourcePath: string | ((id: string) => string),
+  warnings: LocalOverlayConfigWarning[],
+): ConfigInput {
+  let filtered = input;
+
+  for (const backend of CAPLET_BACKEND_KEYS) {
+    const caplets = filtered[backend];
+    if (!isPlainObject(caplets)) {
+      continue;
+    }
+
+    for (const [id, caplet] of Object.entries(caplets)) {
+      const missing = missingEnvReferences(caplet, [backend, id]);
+      if (missing.length === 0) {
+        continue;
+      }
+
+      filtered = removeCapletBackendId(filtered, backend, id);
+      warnings.push({
+        kind,
+        path: typeof sourcePath === "function" ? sourcePath(id) : sourcePath,
+        message: formatMissingEnvWarning(id, missing),
+        recoverable: true,
+      });
+    }
+  }
+
+  return filtered;
+}
+
+function missingEnvReferences(value: unknown, path: string[]): MissingEnvReference[] {
+  if (isPublicMetadataPath(path)) {
+    return [];
+  }
+  if (typeof value === "string") {
+    return missingEnvReferencesInString(value, path.join("."));
+  }
+  if (Array.isArray(value)) {
+    return value.flatMap((item, index) => missingEnvReferences(item, [...path, String(index)]));
+  }
+  if (isPlainObject(value)) {
+    return Object.entries(value).flatMap(([key, nested]) =>
+      missingEnvReferences(nested, [...path, key]),
+    );
+  }
+  return [];
+}
+
+function missingEnvReferencesInString(value: string, path: string): MissingEnvReference[] {
+  const missing: MissingEnvReference[] = [];
+  const pattern = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$env:([A-Za-z_][A-Za-z0-9_]*)/g;
+  for (const match of value.matchAll(pattern)) {
+    const name = match[1] ?? match[2];
+    if (name && process.env[name] === undefined) {
+      missing.push({ name, path });
+    }
+  }
+  return missing;
+}
+
+function formatMissingEnvWarning(id: string, missing: MissingEnvReference[]): string {
+  const names = [...new Set(missing.map((reference) => reference.name))];
+  const paths = [...new Set(missing.map((reference) => reference.path))];
+  const variableLabel = names.length === 1 ? "environment variable" : "environment variables";
+  return `Caplet ${id} references missing ${variableLabel} ${names.join(", ")} at ${paths.join(", ")}; skipping Caplet ${id}.`;
+}
+
 function loadBestEffortCapletFiles(
   root: string,
   kind: ConfigSourceKind,
@@ -1732,7 +1843,17 @@ function loadBestEffortCapletFiles(
   for (const warning of result.warnings) {
     warnings.push({ kind, path: warning.path ?? root, message: warning.message });
   }
-  return { config: result.config, paths: result.paths };
+  const config = quarantineMissingEnvCaplets(
+    result.config,
+    kind,
+    (id) => result.paths[id] ?? root,
+    warnings,
+  );
+  const retainedIds = new Set(capletIds(config));
+  const paths = Object.fromEntries(
+    Object.entries(result.paths).filter(([id]) => retainedIds.has(id)),
+  );
+  return { config, paths };
 }
 
 function errorMessage(error: unknown): string {
@@ -2049,6 +2170,19 @@ function mergeConfigInputsWithSources(...inputs: Array<ConfigInputWithSource | u
   return { input: merged, sources, shadows };
 }
 
+function removeCapletBackendId(
+  input: ConfigInput,
+  backend: (typeof CAPLET_BACKEND_KEYS)[number],
+  id: string,
+): ConfigInput {
+  const caplets = input[backend];
+  if (!isPlainObject(caplets)) {
+    return input;
+  }
+  const { [id]: _removed, ...remaining } = caplets;
+  return { ...input, [backend]: remaining };
+}
+
 function removeCapletId(input: ConfigInput, id: string): ConfigInput {
   const { [id]: _mcpServer, ...mcpServers } = input.mcpServers ?? {};
   const { [id]: _openapiEndpoint, ...openapiEndpoints } = input.openapiEndpoints ?? {};
@@ -2234,16 +2368,7 @@ function interpolateConfig<T>(value: T, path: string[] = []): T {
 }
 
 function isPublicMetadataPath(path: string[]): boolean {
-  if (
-    path.length < 3 ||
-    (path[0] !== "mcpServers" &&
-      path[0] !== "openapiEndpoints" &&
-      path[0] !== "googleDiscoveryApis" &&
-      path[0] !== "graphqlEndpoints" &&
-      path[0] !== "httpApis" &&
-      path[0] !== "cliTools" &&
-      path[0] !== "capletSets")
-  ) {
+  if (path.length < 3 || !CAPLET_BACKEND_KEY_SET.has(path[0] ?? "")) {
     return false;
   }
   return NON_INTERPOLATED_SERVER_FIELDS.has(path[2] ?? "");

packages/core/src/native/service.ts

@@ -1009,14 +1009,16 @@ function createLocalOverlayConfigLoader(options: NativeCapletsServiceOptions) {
       const path = typeof warning.path === "string" ? ` at ${warning.path}` : "";
       writeErr(options, `Caplets local overlay warning${path}: ${warning.message}\n`);
     }
-    const warnings = new Set(result.warnings.map(warningKey));
-    if (hasLoaded && [...warnings].some((warning) => !previousWarnings.has(warning))) {
+    const fatalWarnings = new Set(
+      result.warnings.filter((warning) => !warning.recoverable).map(warningKey),
+    );
+    if (hasLoaded && [...fatalWarnings].some((warning) => !previousWarnings.has(warning))) {
       throw new CapletsError(
         "CONFIG_INVALID",
         "Caplets local overlay reload produced new warnings; keeping last known-good config.",
       );
     }
-    previousWarnings = warnings;
+    previousWarnings = fatalWarnings;
     hasLoaded = true;
     return result.config;
   };