feat(cli): EVAL.yaml → evals.json transpiler (agentv transpile)

apps/cli/src/commands/transpile/index.ts

@@ -0,0 +1,69 @@
+import { writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { command, flag, option, optional, positional, string } from 'cmd-ts';
+
+import { getOutputFilenames, transpileEvalYamlFile } from '@agentv/core';
+
+export const transpileCommand = command({
+  name: 'transpile',
+  description: 'Convert an EVAL.yaml file to Agent Skills evals.json format',
+  args: {
+    input: positional({
+      type: string,
+      displayName: 'input',
+      description: 'Path to EVAL.yaml file',
+    }),
+    outDir: option({
+      type: optional(string),
+      long: 'out-dir',
+      short: 'd',
+      description: 'Output directory (defaults to directory of input file)',
+    }),
+    stdout: flag({
+      long: 'stdout',
+      description: 'Write to stdout instead of file(s) (only valid for single-skill output)',
+    }),
+  },
+  handler: async ({ input, outDir, stdout }) => {
+    let result: ReturnType<typeof transpileEvalYamlFile>;
+    try {
+      result = transpileEvalYamlFile(path.resolve(input));
+    } catch (error) {
+      console.error(`Error: ${(error as Error).message}`);
+      process.exit(1);
+    }
+
+    // Print warnings
+    for (const warning of result.warnings) {
+      console.warn(`Warning: ${warning}`);
+    }
+
+    if (result.files.size === 0) {
+      console.error('Error: No output produced (no tests found)');
+      process.exit(1);
+    }
+
+    if (stdout) {
+      if (result.files.size > 1) {
+        console.error(
+          'Error: --stdout is only valid when input produces a single evals.json (multi-skill input produces multiple files)',
+        );
+        process.exit(1);
+      }
+      const [file] = result.files.values();
+      process.stdout.write(JSON.stringify(file, null, 2));
+      process.stdout.write('\n');
+      return;
+    }
+
+    const outputDir = outDir ? path.resolve(outDir) : path.dirname(path.resolve(input));
+    const fileNames = getOutputFilenames(result);
+
+    for (const [skill, evalsJson] of result.files) {
+      const fileName = fileNames.get(skill) ?? 'evals.json';
+      const outputPath = path.join(outputDir, fileName);
+      writeFileSync(outputPath, `${JSON.stringify(evalsJson, null, 2)}\n`);
+      console.log(`Transpiled to ${outputPath}`);
+    }
+  },
+});

apps/cli/src/index.ts

@@ -10,6 +10,7 @@ import { generateCommand } from './commands/generate/index.js';
 import { initCmdTsCommand } from './commands/init/index.js';
 import { selfCommand } from './commands/self/index.js';
 import { traceCommand } from './commands/trace/index.js';
+import { transpileCommand } from './commands/transpile/index.js';
 import { trimCommand } from './commands/trim/index.js';
 import { validateCommand } from './commands/validate/index.js';
 import { workspaceCommand } from './commands/workspace/index.js';
@@ -29,6 +30,7 @@ export const app = subcommands({
     init: initCmdTsCommand,
     self: selfCommand,
     trace: traceCommand,
+    transpile: transpileCommand,
     trim: trimCommand,
     validate: validateCommand,
     workspace: workspaceCommand,

examples/features/transpile/csv-analyzer.EVAL.yaml

@@ -0,0 +1,26 @@
+tests:
+  - id: csv-top-months
+    criteria: Agent finds the top 3 months by revenue
+    input:
+      - role: user
+        content:
+          - type: file
+            value: evals/files/sales.csv
+          - type: text
+            value: "I have a CSV of monthly sales data. Find the top 3 months by revenue."
+    expected_output: "The top 3 months by revenue are November ($22,500), September ($20,100), and December ($19,400)."
+    assertions:
+      - type: trigger-judge
+        skill: csv-analyzer
+        should_trigger: true
+      - type: rubrics
+        criteria: "Output identifies November as the highest revenue month"
+      - type: contains
+        value: "$22,500"
+
+  - id: irrelevant-query
+    input: "What time is it?"
+    assertions:
+      - type: trigger-judge
+        skill: csv-analyzer
+        should_trigger: false

examples/features/transpile/csv-analyzer.evals.json

@@ -0,0 +1,23 @@
+{
+  "skill_name": "csv-analyzer",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "I have a CSV of monthly sales data. Find the top 3 months by revenue.",
+      "expected_output": "The top 3 months by revenue are November ($22,500), September ($20,100), and December ($19,400).",
+      "files": ["evals/files/sales.csv"],
+      "should_trigger": true,
+      "assertions": [
+        "Agent finds the top 3 months by revenue",
+        "Output identifies November as the highest revenue month",
+        "Output contains '$22,500'"
+      ]
+    },
+    {
+      "id": 2,
+      "prompt": "What time is it?",
+      "should_trigger": false,
+      "assertions": []
+    }
+  ]
+}